Pay attention to some of the following concepts:
/** * @param urlFile Path for file which consists of URLs to be scraped * @param outputFile File where scrape results will be written * @throws InterruptedException * @throws ExecutionException * @throws TimeoutException */public void scrapeURLs(String urlFile, String outputFile) throws InterruptedException, ExecutionException, TimeoutException { Iterator uiter = null; // Get the URLs from the file try { uiter = FileUtil.getURLIterator( urlFile ); } catch (IOException e1) { e1.printStackTrace(); } // Iterate through all URLs if (uiter != null) { // // Create an ExecutorService using a newFixedThreadPool // ExecutorService executorService = Executors.newFixedThreadPool(10); // // Create a map of Future and URLs // Map<Future, String> tasks = new LinkedHashMap<Future, String>(); // Iterator through all URLs for scraping the web while (uiter.hasNext()) { String urlstr = uiter.next(); // // Create a callable instance which calls the function that invokes the scraping for each URL // and get the content (full or part based on some rules) // Callable callable = new Callable() { public String call() throws Exception { return scrapeIndividualURL(urlstr); } }; // // Submit the task to executorService; At this point the scraping starts // Future future = executorService.submit(callable); tasks.put(future, urlstr); } // // For each task, iterate and get the content; Write the content to a file // tasks.forEach((future, url) -> { try { String content = future.get(120, TimeUnit.SECONDS); writeToFile(url, content, outputFile); } catch (InterruptedException | ExecutionException | TimeoutException e) { e.printStackTrace(); writeToFile(url, "Not Found", outputFile); } }); executorService.shutdown(); } } /** * Scrape the URL * @param urlstr * @return */ public static String scrapeIndividualURls( String urlstr ) { URL url = null; StringBuilder contentb = new StringBuilder(); try { // get URL content url = new URL(urlstr); // Create a URL Connection Object HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // Set the configuration parameters // Note the readTimeOut set to 30 seconds. // This is quite important when you are planning to scrape URLs. conn.setConnectTimeout(100000); conn.setReadTimeout(30000); conn.connect(); // open the stream and put it into BufferedReader InputStream in = null; if (conn.getResponseCode() >= 400) { in = conn.getErrorStream(); } else { BufferedReader br = new BufferedReader(new InputStreamReader( conn.getInputStream())); String inputLine; while ((inputLine = br.readLine()) != null) { contentb.append(inputLine); contentb.append("\n"); } br.close(); } } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } return contentb.toString(); } /** * Write to the file * @param url * @param value * @param outputFile */ private void writeToFile(String url, String value, String outputFile) throws IOException { FileWriter fw = new FileWriter( new File( outputFile ), true ); BufferedWriter bw = new BufferedWriter(fw); if( value != null ) { bw.write( url + "\t" + value + "\n" ); } else { bw.write( url + "\t" + "Not Found" + "\n" ); } bw.close(); }
In recent years, artificial intelligence (AI) has evolved to include more sophisticated and capable agents,…
Adaptive learning helps in tailoring learning experiences to fit the unique needs of each student.…
With the increasing demand for more powerful machine learning (ML) systems that can handle diverse…
Anxiety is a common mental health condition that affects millions of people around the world.…
In machine learning, confounder features or variables can significantly affect the accuracy and validity of…
Last updated: 26 Sept, 2024 Credit card fraud detection is a major concern for credit…