From 4a8cb231c3974d3f2000e170ca0c56850bc66c7f Mon Sep 17 00:00:00 2001 From: Matt Singleton Date: Wed, 19 Apr 2017 13:53:33 -0400 Subject: more consistent crawl frequency --- unbiased/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/unbiased/main.py b/unbiased/main.py index f784bce..c8a113e 100755 --- a/unbiased/main.py +++ b/unbiased/main.py @@ -22,11 +22,17 @@ def main(): parser.add_argument('-s', '--scratch', default='/opt/unbiased/scratch', help='writable scratch workspace') args = parser.parse_args() + crawl_frequency = 600 while True: logger.info('Starting crawl') + start = time.time() run(args.webroot, args.scratch) - logger.info('Crawl complete. Sleeping for 600s') - time.sleep(600) + finish = time.time() + runtime = finish - start + sleeptime = crawl_frequency - runtime + logger.info('Crawl complete in {}s. Sleeping for {}s'.format(int(runtime), int(sleeptime))) + if sleeptime > 0: + time.sleep(sleeptime) def run(webroot, scratch): sourceList=[] -- cgit v1.2.3