keep files in memory and only write to disk at the very end

3 files changed, 24 insertions, 15 deletions

diff --git a/unbiased/main.py b/unbiased/main.py
index 949e646..7b057ea 100755
--- a/unbiased/main.py
+++ b/unbiased/main.py
@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 
 import argparse
+import io
 import logging
 import logging.config
 import time
 
-from unbiased.unbiasedFunctions import pickStories, pullImage, buildOutput, writeOutputHTML
+from unbiased.unbiasedFunctions import pickStories, pullImage, buildOutput, write_files, write_static_files
 from unbiased.sources import get_sources
 
 logger = logging.getLogger('unbiased')
@@ -111,20 +112,26 @@ def run(webroot, source_names):
     logger.info('Picked middle stories from: {}'.format([x.source for x in middle_stories]))
     logger.info('Picked bottom stories from: {}'.format([x.source for x in bottom_stories]))
 
+    files_to_write = {}
+
     # download images
     img_idx = 0
     for story in top_stories:
-        story.img = pullImage(story.img, img_idx, webroot, 350, 200)
+        story.img, img_jpg = pullImage(story.img, img_idx, webroot, 350, 200)
+        files_to_write[story.img] = img_jpg
         img_idx += 1
     for story in middle_stories:
-        story.img = pullImage(story.img, img_idx, webroot, 150, 100)
+        story.img, img_jpg =  pullImage(story.img, img_idx, webroot, 150, 100)
+        files_to_write[story.img] = img_jpg
         img_idx += 1
 
-    #build the output file HTML
-    outputHTML = buildOutput(top_stories, middle_stories, bottom_stories)
+    # build the output file HTML
+    output_html = buildOutput(top_stories, middle_stories, bottom_stories)
+    output_html = io.BytesIO(output_html.encode('utf8'))
+    files_to_write['index.html'] = output_html
 
-    #print the output file HTML
-    writeOutputHTML(outputHTML, webroot)
+    write_files(files_to_write, webroot)
+    write_static_files(webroot)
 
 if __name__=="__main__":
     main()
diff --git a/unbiased/sources/base.py b/unbiased/sources/base.py
index 4ff7bf3..68e7f0d 100644
--- a/unbiased/sources/base.py
+++ b/unbiased/sources/base.py
@@ -31,7 +31,6 @@ class NewsSource(object):
        of urls, one for each tier
      - override any of the '_get_*()' functions as necessary
     """
-    # TODO: replace all string parsing with bs4
 
     name = None
     shortname = None
diff --git a/unbiased/unbiasedFunctions.py b/unbiased/unbiasedFunctions.py
index 7825d93..b07245c 100644
--- a/unbiased/unbiasedFunctions.py
+++ b/unbiased/unbiasedFunctions.py
@@ -5,6 +5,7 @@ import os
 import pkgutil

 import random

 import re

+import shutil

 import time

 import urllib.parse

 

@@ -190,12 +191,12 @@ def buildOutput(top_stories, middle_stories, bottom_stories):
     #return updated text

     return html

 

-def writeOutputHTML(outputHTML, outDir):

-    timestamp = time.strftime("%a, %b %-d, %-I:%M%P %Z", time.localtime())

-

-    with open(os.path.join(outDir, 'index.html'), 'w') as fp:

-        fp.write(outputHTML)

+def write_files(files_to_write, outDir):

+    for name, bytesio in files_to_write.items():

+        with open(os.path.join(outDir, name), 'wb') as fp:

+            shutil.copyfileobj(bytesio, fp)

 

+def write_static_files(outDir):

     # copy over static package files

     for filename in ['unbiased.css', 'favicon.ico', 'favicon.png', 'apple-touch-icon.png']:

         data = pkgutil.get_data('unbiased', os.path.join('html_template', filename))

@@ -233,6 +234,8 @@ def pullImage(url, index, webroot, target_width=350, target_height=200):
     img = img.convert('RGB')

     # TODO: create retina images

     jpg_name = 'img{}.jpg'.format(index)

+    jpg_file = io.BytesIO()

     out_file = os.path.join(webroot, jpg_name)

-    img.save(out_file, 'JPEG')

-    return jpg_name

+    img.save(jpg_file, 'JPEG')

+    jpg_file.seek(0)

+    return jpg_name, jpg_file