import os from math import ceil from decimal import Decimal from django.test.client import Client from django.template.loader import render_to_string from django.template import Context from django.core.urlresolvers import reverse from django.apps import apps from django.conf import settings from jsmin import jsmin class _FileWriter(object): """ Given a path and text object; write the page to disc """ def __init__(self, path, text_object, ext='html', filename='index', base_path=settings.NFLATFILES_ROOT): self.path = '%s%s' % (base_path, path) if not os.path.isdir(self.path): os.makedirs(self.path) fpath = '%s%s.%s' % (self.path, filename, ext) self.write(fpath, text_object) if ext == 'js': self.compress_js(filename, text_object) def write(self, fpath, text_object): f = open(fpath, 'wb') f.write(text_object) f.close() def compress_js(self, filename, text_object): path = '%s%s.min.js' % (self.path, filename) compressed = jsmin(text_object.decode(encoding='UTF-8')) self.write(path, compressed) class BuildNew(): def __init__(self, model, app): self.model = apps.get_model(model, app) self.get_model_queryset() self.client = Client() def build(self): self.build_list_view() self.build_detail_view() def get_model_queryset(self): return self.model.objects.filter(status__exact=1) def write_file(self, path, text_object, ext='html', filename='index'): self.writer = _FileWriter(path, text_object, ext=ext, filename=filename) def get_pages(self, qs, paginate_by): return int(ceil(Decimal(qs.count()) / Decimal(paginate_by))) def build_list_view(self, base_path='', qs=None, paginate_by=10): """ Archive Page builder that actually crawls the urls because we need to be able to pass a request object to the template """ if not qs: qs = self.get_model_queryset() pages = self.get_pages(qs, paginate_by) for page in range(pages): if int(pages) > 1: path = '%s%s/' % (base_path, str(page + 1)) url = '%s%s/' % (base_path, str(page + 1)) else: path = base_path url = base_path print(path) response = self.client.get(url, HTTP_HOST='127.0.0.1', follow=True) if page == 0: self.write_file(base_path, response.content) self.write_file(path, response.content) def build_year_view(self, url, paginate_by=99999): years = self.model.objects.dates('pub_date', 'year') for year in years: year = year.strftime('%Y') qs = self.model.objects.filter( status__exact=1, pub_date__year=year ) self.build_list_view( base_path=reverse(url, kwargs={'year': year, }), qs=qs, paginate_by=paginate_by ) def build_month_view(self, url, paginate_by=99999): months = self.model.objects.dates('pub_date', 'month') for m in months: year = m.strftime('%Y') month = m.strftime('%m') qs = self.model.objects.filter( status__exact=1, pub_date__year=year, pub_date__month=month ) if qs.exists(): self.build_list_view( base_path=reverse(url, kwargs={'year': year, 'month': month}), qs=qs, paginate_by=paginate_by ) def build_detail_view(self): ''' Grab all the blog posts, render them to a template string and write that out to the filesystem ''' for entry in self.get_model_queryset(): url = entry.get_absolute_url() path, slug = os.path.split(entry.get_absolute_url()) path = '%s/' % path # write html response = self.client.get(url) self.write_file(path, response.content, filename=slug) # write txt response = self.client.get('%s.txt' % url) self.write_file(path, response.content, ext='txt', filename=slug) # write AMP response = self.client.get('%s.amp' % url) self.write_file(path, response.content, ext='amp', filename=slug) def build_feed(self, url_name): """ Not called, but available for subclassing """ url = reverse(url_name,) path, slug = os.path.split(url) slug, ext = os.path.splitext(slug) response = self.client.get(url, HTTP_HOST='127.0.0.1') self.write_file('%s/' % path, response.content, ext=ext.split(".")[-1], filename=slug) class Build(): def write_file(self, path, text_object, ext='html', filename='index'): """ Given a path and object intended to be a webpage, write the page the disc """ path = '%s%s' % (settings.FLATFILES_ROOT, path) if not os.path.isdir(path): os.makedirs(path) fpath = '%s%s.%s' % (path, filename, ext) file = open(fpath, 'wb') file.write(text_object) file.close() if ext == 'js': from jsmin import jsmin fpath = '%s%s.min.%s' % (path, filename, ext) compressed = jsmin(text_object.decode(encoding='UTF-8')) with open(fpath, 'wb') as js_file: minified = js_file.write(compressed.encode('utf-8')) js_file.close() def build_archive_pages(self, qs=None, base_path='', paginate_by=10): """ Archive Page builder that actually crawls the urls because we need to be able to pass a request object to the template """ if qs is None: qs = self.get_model_querset() c = Client() pages = ceil(Decimal(qs.count()) / Decimal(paginate_by)) for page in range(int(pages)): path = '%s%s/' % (base_path, page + 1) url = '/%s%s/' % (base_path, str(page + 1)) page_url = base_path + '%d/' response = c.post(url, {'page_url': page_url, 'page': int(page), 'builder': True}, HTTP_HOST='127.0.0.1') if page == 0: self.write_file(base_path, response.content) self.write_file(path, response.content) class BuildAll(Build): def build(self): BuildWriting().build() BuildPhotos().build() BuildProjects().build() BuildMap().build() BuildWritingFeed().build() BuildSitemap().build() BuildPages().build() p.write_files() class BuildWriting(Build): def build(self): self.build_detail_pages() self.build_writing_archives() self.build_country_archive_pages() self.build_region_archive_pages() self.build_homepage() self.build_404() self.writing_year_archives() self.writing_month_archives() def get_model_querset(self): model = apps.get_model('jrnl', 'entry') qs = model.objects.filter(status__exact=1) return qs def build_detail_pages(self): ''' Grab all the blog posts, render them to a template string and write that out to the filesystem ''' qs = self.get_model_querset() for entry in qs: c = Context({'object': entry, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL}) t = render_to_string('details/entry.html', c).encode('utf-8') path = '/jrnl/%s/' %(entry.pub_date.strftime("%Y/%m").lower()) slug = '%s' %(entry.slug) self.write_file(path, t, 'html', slug) s = render_to_string('details/entry.txt',c).encode('utf-8') self.write_file(path, s,'txt', slug) def build_writing_archives(self): qs = self.get_model_querset() self.build_archive_pages(qs, 'jrnl/') def build_region_archive_pages(self): model = apps.get_model('locations', 'Region') blog = apps.get_model('jrnl', 'entry') regions = model.objects.all() for c in regions: qs = blog.objects.filter(status__exact=1, location__state__country__lux_region=c.id).order_by('-pub_date') path = 'jrnl/%s/' % (c.slug) self.build_archive_pages(qs, path) def build_country_archive_pages(self): model = apps.get_model('locations', 'Country') blog = apps.get_model('jrnl', 'entry') countries = model.objects.filter(visited=True) for c in countries: qs = blog.objects.filter(status__exact=1, location__state__country=c).order_by('-pub_date') path = 'jrnl/%s/' % (c.slug) self.build_archive_pages(qs, path) def writing_year_archives(self): entry = apps.get_model('jrnl', 'entry') years = entry.objects.dates('pub_date', 'year') for year in years: year = year.strftime('%Y') qs = entry.objects.filter(status__exact=1, pub_date__year=year).order_by('pub_date') c = Context({'type': 'year', 'year': year, 'object_list': qs}) t = render_to_string('archives/writing_date.html', c).encode('utf-8') fpath = 'jrnl/%s/' % (year) self.write_file(fpath, t) def writing_month_archives(self): entry = apps.get_model('jrnl', 'entry') months = entry.objects.dates('pub_date', 'month') for m in months: year = m.strftime('%Y') month = m.strftime('%m') month_name = m.strftime('%b') month_full_name = m.strftime('%B') qs = entry.objects.filter(status__exact=1, pub_date__year=year, pub_date__month=month).order_by('pub_date') c = Context({'type': 'monthly', 'year': year, 'month': month_full_name, 'object_list': qs, }) t = render_to_string('archives/writing_date.html', c).encode('utf-8') fpath = 'jrnl/%s/%s/' % (year, month) self.write_file(fpath, t) def build_homepage(self): obj = apps.get_model('blog', 'homepagecurrator').objects.get(pk=1) recent = apps.get_model('blog', 'entry').objects.filter(status__exact=1)[:4] template = obj.template_name c = Context({'homepage': obj, 'recent': recent, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL}) t = render_to_string(template, c).encode('utf-8') self.write_file('', t) def build_404(self): c = Context() t = render_to_string('404.html', c).encode('utf-8') self.write_file('', t, 'html', '404') class BuildPhotos(Build): def build(self): self.build_photo_archive_pages() self.build_detail_pages() self.build_js() def build_photo_archive_pages(self): qs = apps.get_model('photos', 'PhotoGallery').objects.all() self.build_archive_pages(qs, 'photos/', 18) def build_detail_pages(self): qs = apps.get_model('photos', 'PhotoGallery').objects.all() for photo in qs: c = Context({'object': photo, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL}) t = render_to_string('details/photo_galleries.html', c).encode('utf-8') path = 'photos/galleries/%s/' % (photo.set_slug) self.write_file(path, t) def build_js(self): fpath = '%sdesign/templates/js/leaflet-providers.js' % settings.PROJ_ROOT leaflet_providers_js = open(fpath, 'r').read() fpath = '%sapp/photos/photos.js' % settings.PROJ_ROOT photos_js = open(fpath, 'r', encoding='UTF8').read() js = leaflet_providers_js + photos_js self.write_file('media/js/', js.encode('utf-8'), 'js', 'photos') class BuildProjects(Build): def build(self): self.build_project_archive() self.build_project_details() self.build_project_data() self.build_gifs() self.build_np_basejs() def get_projects(self): all_proj = [] projects = apps.get_model('projects', 'Project').objects.get(pk=2) for proj in projects: row = {'slug': proj.slug, 'name': proj.model_name} all_proj.append(row) return all_proj def build_project_archive(self): qs = apps.get_model('projects', 'Project').objects.filter(status__exact=1).order_by('-pub_date') c = Context({'object_list': qs, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL}) t = render_to_string('archives/projects.html', c).encode('utf-8') self.write_file('projects/', t) def build_project_details(self): projects = self.get_projects() for proj in projects: model = apps.get_model('projects', proj['name']) if proj['name'] == 'NationalParks': qs = model.objects.filter(visited__exact=True).order_by("-date_visited_begin") else: qs = model.objects.filter(status__exact=1) c = Context({ 'object_list': qs, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL }) t = render_to_string('details/%s.html' % (proj['slug']), c).encode('utf-8') path = 'projects/%s/' % (proj['slug']) self.write_file(path, t) """ not sure how to handle projects really, the above doesn't work and if I just keep writing if/else statements that gets messy, so I guess functions it is. """ def build_gifs(self): qs = apps.get_model('projects', 'AnimatedGif').objects.all() for gif in qs: c = Context({ 'object': gif, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL }) t = render_to_string('details/gifs.html', c).encode('utf-8') path = 'projects/gifs/%s/' % (gif.slug) self.write_file(path, t) def build_project_data(self): model = apps.get_model('projects', 'NationalParks') for park in model.objects.filter(visited__exact=True): path = 'projects/data/natparks/' json = park.mpoly.json self.write_file(path, json.encode('utf-8'), 'json', park.id) def build_np_basejs(self): fpath = '%sdesign/templates/js/leaflet-providers.js' % settings.PROJ_ROOT leaflet_providers_js = open(fpath, 'r').read() fpath = '%sapp/projects/natparks.js' % settings.PROJ_ROOT natparks_js = open(fpath, 'r').read() js = leaflet_providers_js + natparks_js self.write_file('media/js/', js.encode('utf-8'), 'js', 'natparks') class BuildSitemap(Build): def build(self): c = Client() response = c.get('/sitemap.xml', HTTP_HOST='127.0.0.1') self.write_file('', response.content, 'xml', 'sitemap') class BuildWritingFeed(Build): def build(self): qs = apps.get_model('blog', 'entry').objects.filter(status__exact=1).order_by('-pub_date')[:20] c = Context({'object_list': qs, 'SITE_URL': settings.SITE_URL}) t = render_to_string('feed.xml', c).encode('utf-8') fpath = '%s' % ('rss/',) self.write_file(fpath, t, 'xml') class BuildPages(Build): def build(self): model = apps.get_model('pages', 'page') pages = model.objects.all() for page in pages: c = Context({'object':page,'SITE_URL':settings.SITE_URL, 'MEDIA_URL':settings.BAKED_MEDIA_URL}) t = render_to_string(["details/%s.html" % page.slug, 'details/page.html'],c).encode('utf-8') s = render_to_string('details/page.txt',c).encode('utf-8') fpath = '%s' %(page.slug) self.write_file('', t, 'html', page.slug) self.write_file('', t, 'txt', page.slug) class BuildMap(Build): def build(self): qs = apps.get_model('jrnl', 'entry').objects.filter(status__exact=1) cl = apps.get_model('locations', 'Country').objects.filter(visited=True).exclude(name='default') rl = apps.get_model('locations', 'Region').objects.all() rtl = apps.get_model('locations', 'Route').objects.all() c = Context({ 'object_list': qs, 'country_list': cl, 'region_list': rl, 'route_list': rtl, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL }) t = render_to_string('archives/map_data.html', c).encode('utf-8') fpath = '%sdesign/templates/js/leaflet-providers.js' % settings.PROJ_ROOT leaflet_providers_js = open(fpath, 'r').read() js = leaflet_providers_js + t.decode(encoding='utf-8') self.write_file('media/js/', js.encode('utf-8'), 'js', 'mainmap') c = Context({ 'country_list': cl, 'region_list': rl, 'route_list': rtl, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL }) t = render_to_string('archives/map.html', c).encode('utf-8') self.write_file('', t, "html",'map') # Back up entries to markdown text files which are then stored in dropbox and git class EntryBak(Build): def get_model_querset(self): model = apps.get_model('jrnl', 'entry') qs = model.objects.filter(status__exact=1) return qs def write_file(self, path, text_object): file = open(path, 'wb') file.write(text_object) file.close() def build_writing_bak(self): qs = self.get_model_querset() for obj in qs: c = Context({'object': obj, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL}) path = "%szbak/posts/%s_%s.txt" %(settings.PROJ_ROOT, (obj.pub_date.strftime("%Y-%m-%d").lower()), obj.slug) t = render_to_string('details/entry-bak.txt', c).encode('utf-8') self.write_file(path, t) class BuildBooks(Build): def build(self): self.build_detail_pages() self.build_book_archive_pages() def build_book_archive_pages(self): qs = apps.get_model('books', 'Book').objects.all().order_by('-read_date').select_related() print(qs) self.build_archive_pages(qs, 'books/', 18) def build_detail_pages(self): qs = apps.get_model('books', 'Book').objects.all().order_by('-read_date').select_related() for book in qs: c = Context({'object': book,}) t = render_to_string('details/book.html', c).encode('utf-8') path = 'books/' slug = '%s' % (book.slug) self.write_file(path, t, 'html', slug)