From 660ae0943b614a49bd988fd61d701a52e9daef71 Mon Sep 17 00:00:00 2001 From: Matt Jackson Date: Thu, 17 Jul 2014 15:32:22 +0100 Subject: [PATCH 1/2] ignore .env folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bea68953..376187c8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ env/ .idea ._* venv/ +.env/ goose_extractor.egg-info/ From edab74bfea0c0d461c2c4266cc053f0e08b7dd7e Mon Sep 17 00:00:00 2001 From: Matt Jackson Date: Thu, 17 Jul 2014 15:37:34 +0100 Subject: [PATCH 2/2] decoupled http operations & document processing/extraction so we can use our own crawlers and use goose only for post processing --- goose/crawler.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/goose/crawler.py b/goose/crawler.py index 211d410e..dcb883ec 100644 --- a/goose/crawler.py +++ b/goose/crawler.py @@ -83,6 +83,10 @@ def crawl(self, crawl_candidate): # raw html raw_html = self.get_html(crawl_candidate, parse_candidate) + return self.process(parse_candidate.url, parse_candidate.link_hash, raw_html) + + def process(self, final_url, link_hash, raw_html): + if raw_html is None: return self.article @@ -90,8 +94,8 @@ def crawl(self, crawl_candidate): doc = self.get_document(raw_html) # article - self.article.final_url = parse_candidate.url - self.article.link_hash = parse_candidate.link_hash + self.article.final_url = final_url + self.article.link_hash = link_hash self.article.raw_html = raw_html self.article.doc = doc self.article.raw_doc = deepcopy(doc)