diff options
| -rwxr-xr-x | app/blog/parse.py | 52 | ||||
| -rw-r--r-- | app/books/__init__.py | 0 | ||||
| -rw-r--r-- | app/books/models.py | 20 | ||||
| -rw-r--r-- | app/stuff/__init__.py | 0 | ||||
| -rw-r--r-- | app/stuff/admin.py | 9 | ||||
| -rw-r--r-- | app/stuff/models.py | 43 | ||||
| -rw-r--r-- | app/stuff/temp.py | 114 |
7 files changed, 52 insertions, 186 deletions
diff --git a/app/blog/parse.py b/app/blog/parse.py new file mode 100755 index 0000000..27f5165 --- /dev/null +++ b/app/blog/parse.py @@ -0,0 +1,52 @@ +#!/usr/bin/python +import os, datetime +from dateutil.parser import parse as dateparser +from os.path import abspath,dirname +from django.core.exceptions import ObjectDoesNotExist +from blog.models import Entry + +from django.conf import settings + +def parse_file(filepath): + data = {} + contents = open(filepath).read() + raw = contents.splitlines() + for line in raw[1:]: + if line == '---': + break + else: + k,v = line.split(':', 1) + data[k.strip()] = v.strip() + body = "\n".join(line.strip() for line in raw[1:]) + data["body_markdown"] = body.split('---')[1] + return data + +""" + now I need a function to query the db for the title and date + if there's no entry then it's new and we add it and publish + What about edits though? Crap, edits. That means we need to check lastmod + and that's notoriously inaccurate. damn. +from blog.parse import * +crawl_dir() + +""" +def crawl_dir(): + file_root = settings.POSTS_DIR + file_list = os.listdir(file_root) + file_list = filter(lambda item: not (item.startswith('README') or item.startswith('updategithub.php') or item.startswith('.') or item.endswith('~')),file_list) + for f in file_list: + fpath = file_root+"/"+f + last_mod = datetime.datetime.fromtimestamp(os.path.getmtime(fpath)) + last_run = datetime.datetime.fromtimestamp(os.path.getmtime(abspath(dirname(__file__))+'/last_run')) + if last_mod > last_run: + print "needs an update" + data = parse_file(fpath) + date = dateparser(data['pub_date']) + try: + row = Entry.objects.get(title=str(data['title']),pub_date=date) + print row.title, date + except ObjectDoesNotExist: + print data['title'] + str(date) + " = not found" + last_mod_dump = open(abspath(dirname(__file__))+'/last_run','w') + print last_mod_dump + print >> last_mod_dump, str(datetime.datetime.now()) diff --git a/app/books/__init__.py b/app/books/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/app/books/__init__.py +++ /dev/null diff --git a/app/books/models.py b/app/books/models.py deleted file mode 100644 index 4e28b3d..0000000 --- a/app/books/models.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.db import models - -class Book(models.Model): - id = models.IntegerField(primary_key=True) - title = models.CharField(max_length=200) - author_name = models.CharField(max_length=200) - slug = models.CharField(max_length=50) - url = models.CharField(max_length=400) - year_pub = models.CharField(max_length=5) - read_date = models.DateTimeField() - isbn = models.CharField(max_length=100) - body_html = models.TextField() - pub_date = models.DateTimeField() - tags = models.CharField(max_length=255) - rating = models.CharField(max_length=1) - enable_comments = models.BooleanField() - status = models.IntegerField() - - - diff --git a/app/stuff/__init__.py b/app/stuff/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/app/stuff/__init__.py +++ /dev/null diff --git a/app/stuff/admin.py b/app/stuff/admin.py deleted file mode 100644 index 6e8a27d..0000000 --- a/app/stuff/admin.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.contrib import admin -from stuff.models import Item - - -class ItemAdmin(admin.ModelAdmin): - list_display = ('title', 'image_url','admin_thumbnail','creator','type', 'current_price', 'format','isbn') - list_filter = ['format', 'type'] - -admin.site.register(Item, ItemAdmin) diff --git a/app/stuff/models.py b/app/stuff/models.py deleted file mode 100644 index 70b7366..0000000 --- a/app/stuff/models.py +++ /dev/null @@ -1,43 +0,0 @@ -from django.db import models -from django.utils.encoding import force_unicode -from django.conf import settings -ITEM_TYPES = ( - (0, 'Book'), - (1, 'Apparel'), - (2, 'Gadget'), - (3, 'Furniture'), - ) - -class Item(models.Model): - isbn = models.CharField(max_length=100, blank=True, null=True) - serial_number = models.CharField(max_length=100, blank=True, null=True) - type = models.IntegerField(choices=ITEM_TYPES, default=0) - title = models.CharField(max_length=200) - creator = models.CharField(max_length=200, blank=True, null=True) - creator_sort_last_name = models.CharField(max_length=200, blank=True, null=True) - creator_sort_first_name = models.CharField(max_length=200, blank=True, null=True) - current_price = models.DecimalField(max_digits=8, decimal_places=2, blank=True, null=True) - retail_price = models.DecimalField(max_digits=8, decimal_places=2, blank=True, null=True) - purchase_price = models.DecimalField(max_digits=8, decimal_places=2, blank=True, null=True) - amazon_url = models.CharField(max_length=400,blank=True, null=True) - image_url = models.CharField(max_length=400,blank=True, null=True) - image = models.FileField(upload_to='stuff/', null=True,blank=True) - format = models.CharField(max_length=255, blank=True, null=True) - date_released = models.DateTimeField(blank=True, null=True) - date_purchased = models.DateTimeField(blank=True, null=True) - date_added = models.DateTimeField(blank=True, null=True) - - class Meta: - ordering = ['-date_added'] - - def __unicode__(self): - return self.title - - def get_image_url(self): - return '%s%s' %(settings.MEDIA_URL,self.image) - - def admin_thumbnail(self): - return force_unicode('<a href=""><img src="%s" width="100" style="width:100px"></a>' % (self.get_image_url())) - admin_thumbnail.allow_tags = True - admin_thumbnail.short_description = 'Thumbnail' - diff --git a/app/stuff/temp.py b/app/stuff/temp.py deleted file mode 100644 index 7bf49b7..0000000 --- a/app/stuff/temp.py +++ /dev/null @@ -1,114 +0,0 @@ -from stuff.models import Item -from django.conf import settings -from datetime import datetime -from decimal import Decimal -import csv -item_types = { 'Book':0, 'Apparel': 1,'Gadget': 2, 'Furniture': 3} -""" -csv_file = csv.DictReader(open('stuff.csv', 'rb'), delimiter=',') -for line in csv_file: - if line['release date'][:-6] != '': - date_released = datetime.strptime(line['release date'][:-6], "%Y-%m-%d %H:%M:%S") - else: - date_released = None - if line['purchase date'][:-6] != '': - date_purchased = datetime.strptime(line['purchase date'][:-6], "%Y-%m-%d %H:%M:%S") - else: - date_purchased = None - if line['creation date'][:-6] != '': - date_added = datetime.strptime(line['creation date'][:-6], "%Y-%m-%d %H:%M:%S") - else: - date_added = None - item_type = int(item_types[line['item type']]) - if line['current value'][1:] != '': - try: - current_price = Decimal(line['current value'][1:].replace(",", "")) - except: - current_price = None - else: - current_price = None - if line['retail price'][1:] != '': - try: - retail_price = Decimal(line['retail price'][1:].replace(",", "")) - except: - retail_price =None - else: - retail_price =None - if line['purchase price'][1:] != '' and line['purchase price'] != "Gift": - try: - purchase_price = Decimal(line['purchase price'][1:].replace(",", "").replace('\\xa0', ' ')) - except: - purchase_price =None - else: - purchase_price =None - i, created = Item.objects.get_or_create( - isbn = line['ISBN'], - serial_number = line['serial number'], - type = item_type, - title = line['title'], - creator = line['creator'], - current_price = current_price, - retail_price = retail_price, - purchase_price = purchase_price, - amazon_url = line['amazon link'], - image_url = line['coverImageLargeURLString'], - format = line['format'], - date_released = date_released, - date_purchased = date_purchased, - date_added = date_added - ) - print created, i.title -""" -import datetime -import os -import cStringIO # *much* faster than StringIO -import urllib - - -from django.contrib.contenttypes.models import ContentType -from django.template.defaultfilters import slugify -from django.core.exceptions import ObjectDoesNotExist -from django.utils.encoding import force_unicode,smart_unicode -from django.conf import settings - -# Required PIL classes may or may not be available from the root namespace -# depending on the installation -try: - import Image - import ImageFile - import ImageFilter - import ImageEnhance -except ImportError: - try: - from PIL import Image - from PIL import ImageFile - from PIL import ImageFilter - from PIL import ImageEnhance - except ImportError: - raise ImportError("Could not import the Python Imaging Library.") - -ImageFile.MAXBLOCK = 1000000 - - -def grab_item_image(item): - crop_dir = settings.IMAGES_ROOT + '/stuff/' - if not os.path.isdir(crop_dir): - os.makedirs(crop_dir) - remote = item.image_url - print remote - fname = urllib.urlopen(remote) - im = cStringIO.StringIO(fname.read()) # constructs a StringIO holding the image - img = Image.open(im) - filename = '%s/%s.jpg' %(crop_dir, item.id) - print filename - try: - if img.format == 'JPEG': - img.save(filename, 'JPEG', quality=95, optimize=True) - else: - img.save(filename) - except IOError: - if os.path.isfile(filename): - os.unlink(filename) - pass - return 'images/stuff/%s.jpg'%(item.id) - |