path: root/app/builder/base.py

import os
from math import ceil
from decimal import Decimal
from django.test.client import Client
from django.template.loader import render_to_string
from django.template import Context
from django.core.urlresolvers import reverse
from django.apps import apps
from django.conf import settings
from jsmin import jsmin


class _FileWriter(object):
    """
    Given a path and text object; write the page to disc
    """
    def __init__(self, path, text_object, ext='html', filename='index', base_path=settings.NFLATFILES_ROOT):
        self.path = '%s%s' % (base_path, path)
        if not os.path.isdir(self.path):
            os.makedirs(self.path)
        fpath = '%s%s.%s' % (self.path, filename, ext)
        self.write(fpath, text_object)
        if ext == 'js':
            self.compress_js(filename, text_object)

    def write(self, fpath, text_object):
        f = open(fpath, 'wb')
        f.write(text_object)
        f.close()

    def compress_js(self, filename, text_object):
        path = '%s%s.min.js' % (self.path, filename)
        compressed = jsmin(text_object.decode(encoding='UTF-8'))
        self.write(path, compressed)


class BuildNew():
    def __init__(self, model, app):
        self.model = apps.get_model(model, app)
        self.get_model_queryset()
        self.client = Client()

    def build(self):
        self.build_list_view()
        self.build_detail_view()

    def get_model_queryset(self):
        return self.model.objects.filter(status__exact=1)

    def write_file(self, path, text_object, ext='html', filename='index'):
        self.writer = _FileWriter(path, text_object, ext=ext, filename=filename)

    def get_pages(self, qs, paginate_by):
        return int(ceil(Decimal(qs.count()) / Decimal(paginate_by)))

    def build_list_view(self, base_path='', qs=None, paginate_by=10):
        """
        Archive Page builder that actually crawls the urls
        because we need to be able to pass a request object to the template
        """

        if not qs:
            qs = self.get_model_queryset()
        pages = self.get_pages(qs, paginate_by)
        for page in range(pages):
            if int(pages) > 1:
                path = '%s%s/' % (base_path, str(page + 1))
                url = '%s%s/' % (base_path, str(page + 1))
            else:
                path = base_path
                url = base_path
            print(path)
            response = self.client.get(url, HTTP_HOST='127.0.0.1', follow=True)
            if page == 0:
                self.write_file(base_path, response.content)
            self.write_file(path, response.content)

    def build_year_view(self, url, paginate_by=99999):
        years = self.model.objects.dates('pub_date', 'year')
        for year in years:
            year = year.strftime('%Y')
            qs = self.model.objects.filter(
                status__exact=1,
                pub_date__year=year
            )
            self.build_list_view(
                base_path=reverse(url, kwargs={'year': year, }),
                qs=qs,
                paginate_by=paginate_by
            )

    def build_month_view(self, url, paginate_by=99999):
        months = self.model.objects.dates('pub_date', 'month')
        for m in months:
            year = m.strftime('%Y')
            month = m.strftime('%m')
            qs = self.model.objects.filter(
                status__exact=1,
                pub_date__year=year,
                pub_date__month=month
            )
            if qs.exists():
                self.build_list_view(
                    base_path=reverse(url, kwargs={'year': year, 'month': month}),
                    qs=qs,
                    paginate_by=paginate_by
                )

    def build_detail_view(self):
        '''
        Grab all the blog posts, render them to a template
        string and write that out to the filesystem
        '''
        for entry in self.get_model_queryset():
            url = entry.get_absolute_url()
            path, slug = os.path.split(entry.get_absolute_url())
            path = '%s/' % path
            # write html
            response = self.client.get(url)
            self.write_file(path, response.content, filename=slug)
            # write txt
            response = self.client.get('%s.txt' % url)
            self.write_file(path, response.content, ext='txt', filename=slug)
            # write AMP
            response = self.client.get('%s.amp' % url)
            self.write_file(path, response.content, ext='amp', filename=slug)

    def build_feed(self, url_name):
        """
        Not called, but available for subclassing
        """
        url = reverse(url_name,)
        path, slug = os.path.split(url)
        slug, ext = os.path.splitext(slug)
        response = self.client.get(url, HTTP_HOST='127.0.0.1')
        self.write_file('%s/' % path, response.content, ext=ext.split(".")[-1], filename=slug)

class Build():

    def write_file(self, path, text_object, ext='html', filename='index'):
        """
        Given a path and object intended to be a webpage, write the page the
        disc
        """
        path = '%s%s' % (settings.FLATFILES_ROOT, path)
        if not os.path.isdir(path):
            os.makedirs(path)
        fpath = '%s%s.%s' % (path, filename, ext)
        file = open(fpath, 'wb')
        file.write(text_object)
        file.close()
        if ext == 'js':
            from jsmin import jsmin
            fpath = '%s%s.min.%s' % (path, filename, ext)
            compressed = jsmin(text_object.decode(encoding='UTF-8'))
            with open(fpath, 'wb') as js_file:
                minified = js_file.write(compressed.encode('utf-8'))
                js_file.close()

    def build_archive_pages(self, qs=None, base_path='', paginate_by=10):
        """
        Archive Page builder that actually crawls the urls
        because we need to be able to pass a request object to the template

        """
        if qs is None:
            qs = self.get_model_querset()
        c = Client()
        pages = ceil(Decimal(qs.count()) / Decimal(paginate_by))
        for page in range(int(pages)):
            path = '%s%s/' % (base_path, page + 1)
            url = '/%s%s/' % (base_path, str(page + 1))
            page_url = base_path + '%d/'
            response = c.post(url, {'page_url': page_url, 'page': int(page), 'builder': True}, HTTP_HOST='127.0.0.1')
            if page == 0:
                self.write_file(base_path, response.content)
            self.write_file(path, response.content)

class BuildAll(Build):
    def build(self):
        BuildWriting().build()
        BuildPhotos().build()
        BuildProjects().build()
        BuildMap().build()
        BuildWritingFeed().build()
        BuildSitemap().build()
        BuildPages().build()
        p.write_files()

class BuildWriting(Build):
    def build(self):
        self.build_detail_pages()
        self.build_writing_archives()
        self.build_country_archive_pages()
        self.build_region_archive_pages()
        self.build_homepage()
        self.build_404()
        self.writing_year_archives()
        self.writing_month_archives()

    def get_model_querset(self):
        model = apps.get_model('jrnl', 'entry')
        qs = model.objects.filter(status__exact=1)
        return qs

    def build_detail_pages(self):
        '''
        Grab all the blog posts, render them to a template string and write that out to the filesystem
        '''
        qs = self.get_model_querset()
        for entry in qs:
            c = Context({'object': entry, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL})
            t = render_to_string('details/entry.html', c).encode('utf-8')
            path = '/jrnl/%s/' %(entry.pub_date.strftime("%Y/%m").lower())
            slug = '%s' %(entry.slug)
            self.write_file(path, t, 'html', slug)
            s = render_to_string('details/entry.txt',c).encode('utf-8') 
            self.write_file(path, s,'txt', slug)

    def build_writing_archives(self):
        qs = self.get_model_querset()
        self.build_archive_pages(qs, 'jrnl/')

    def build_region_archive_pages(self):
        model = apps.get_model('locations', 'Region')
        blog = apps.get_model('jrnl', 'entry')
        regions = model.objects.all()
        for c in regions:
            qs = blog.objects.filter(status__exact=1, location__state__country__lux_region=c.id).order_by('-pub_date')
            path = 'jrnl/%s/' % (c.slug)
            self.build_archive_pages(qs, path)

    def build_country_archive_pages(self):
        model = apps.get_model('locations', 'Country')
        blog = apps.get_model('jrnl', 'entry')
        countries = model.objects.filter(visited=True)
        for c in countries:
            qs = blog.objects.filter(status__exact=1, location__state__country=c).order_by('-pub_date')
            path = 'jrnl/%s/' % (c.slug)
            self.build_archive_pages(qs, path)

    def writing_year_archives(self):
        entry = apps.get_model('jrnl', 'entry')
        years = entry.objects.dates('pub_date', 'year')
        for year in years:
            year = year.strftime('%Y')
            qs = entry.objects.filter(status__exact=1, pub_date__year=year).order_by('pub_date')
            c = Context({'type': 'year', 'year': year, 'object_list': qs})
            t = render_to_string('archives/writing_date.html', c).encode('utf-8')
            fpath = 'jrnl/%s/' % (year)
            self.write_file(fpath, t)

    def writing_month_archives(self):
        entry = apps.get_model('jrnl', 'entry')
        months = entry.objects.dates('pub_date', 'month')
        for m in months:
            year = m.strftime('%Y')
            month = m.strftime('%m')
            month_name = m.strftime('%b')
            month_full_name = m.strftime('%B')
            qs = entry.objects.filter(status__exact=1, pub_date__year=year,
                                      pub_date__month=month).order_by('pub_date')
            c = Context({'type': 'monthly', 'year': year, 'month': month_full_name, 'object_list': qs, })
            t = render_to_string('archives/writing_date.html', c).encode('utf-8')
            fpath = 'jrnl/%s/%s/' % (year, month)
            self.write_file(fpath, t)

    def build_homepage(self):
        obj = apps.get_model('blog', 'homepagecurrator').objects.get(pk=1)
        recent = apps.get_model('blog', 'entry').objects.filter(status__exact=1)[:4]
        template = obj.template_name
        c = Context({'homepage': obj, 'recent': recent, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL})
        t = render_to_string(template, c).encode('utf-8')
        self.write_file('', t)

    def build_404(self):
        c = Context()
        t = render_to_string('404.html', c).encode('utf-8')
        self.write_file('', t, 'html', '404')

class BuildPhotos(Build):
    def build(self):
        self.build_photo_archive_pages()
        self.build_detail_pages()
        self.build_js()

    def build_photo_archive_pages(self):
        qs = apps.get_model('photos', 'PhotoGallery').objects.all()
        self.build_archive_pages(qs, 'photos/', 18)

    def build_detail_pages(self):
        qs = apps.get_model('photos', 'PhotoGallery').objects.all()
        for photo in qs:
            c = Context({'object': photo, 'MEDIA_URL':
                        settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL})
            t = render_to_string('details/photo_galleries.html', c).encode('utf-8')
            path = 'photos/galleries/%s/' % (photo.set_slug)
            self.write_file(path, t)
    
    def build_js(self):
        fpath = '%sdesign/templates/js/leaflet-providers.js' % settings.PROJ_ROOT
        leaflet_providers_js = open(fpath, 'r').read()
        fpath = '%sapp/photos/photos.js' % settings.PROJ_ROOT
        photos_js = open(fpath, 'r', encoding='UTF8').read()
        js = leaflet_providers_js + photos_js 
        self.write_file('media/js/', js.encode('utf-8'), 'js', 'photos')

class BuildProjects(Build):
    def build(self):
        self.build_project_archive()
        self.build_project_details()
        self.build_project_data()
        self.build_gifs()
        self.build_np_basejs()

    def get_projects(self):
        all_proj = []
        proj = apps.get_model('projects', 'Project').objects.get(pk=2)
        row = {'slug': proj.slug, 'name': proj.model_name}
        all_proj.append(row)
        return all_proj

    def build_project_archive(self):
        qs = apps.get_model('projects', 'Project').objects.filter(status__exact=1).order_by('-pub_date')
        c = Context({'object_list': qs, 'MEDIA_URL': settings.BAKED_MEDIA_URL,
                    'IMAGES_URL': settings.BAKED_IMAGES_URL})
        t = render_to_string('archives/projects.html', c).encode('utf-8')
        self.write_file('projects/', t)

    def build_project_details(self):
        projects = self.get_projects()
        for proj in projects:
            model = apps.get_model('projects', proj['name'])
            if proj['name'] == 'NationalParks':
                qs = model.objects.filter(visited__exact=True).order_by("-date_visited_begin")
            else:
                qs = model.objects.filter(status__exact=1)
            c = Context({
                'object_list': qs,
                'MEDIA_URL': settings.BAKED_MEDIA_URL,
                'IMAGES_URL': settings.BAKED_IMAGES_URL
            })
            t = render_to_string('details/%s.html' % (proj['slug'].split("/")[1]), c).encode('utf-8')
            path = 'projects/%s/' % (proj['slug'].split("/")[1])
            self.write_file(path, t)

    """
    not sure how to handle projects really, the above doesn't work and
    if I just keep writing if/else statements that gets messy, so I guess
    functions it is.
    """
    def build_gifs(self):
        qs = apps.get_model('projects', 'AnimatedGif').objects.all()
        for gif in qs:
            c = Context({
                'object': gif,
                'MEDIA_URL': settings.BAKED_MEDIA_URL,
                'IMAGES_URL': settings.BAKED_IMAGES_URL
            })
            t = render_to_string('details/gifs.html', c).encode('utf-8')
            path = 'projects/gifs/%s/' % (gif.slug)
            self.write_file(path, t)

    def build_project_data(self):
        model = apps.get_model('projects', 'NationalParks')
        for park in model.objects.filter(visited__exact=True):
            path = 'projects/data/natparks/'
            json = park.mpoly.json
            self.write_file(path, json.encode('utf-8'), 'json', park.id)

    def build_np_basejs(self):
        fpath = '%sdesign/templates/js/leaflet-providers.js' % settings.PROJ_ROOT
        leaflet_providers_js = open(fpath, 'r').read()
        fpath = '%sapp/projects/natparks.js' % settings.PROJ_ROOT
        natparks_js = open(fpath, 'r').read()
        js = leaflet_providers_js + natparks_js
        self.write_file('media/js/', js.encode('utf-8'), 'js', 'natparks')


class BuildSitemap(Build):
    def build(self):
        c = Client()
        response = c.get('/sitemap.xml', HTTP_HOST='127.0.0.1')
        self.write_file('', response.content, 'xml', 'sitemap')


class BuildWritingFeed(Build):
    def build(self):
        qs = apps.get_model('blog', 'entry').objects.filter(status__exact=1).order_by('-pub_date')[:20]
        c = Context({'object_list': qs, 'SITE_URL': settings.SITE_URL})
        t = render_to_string('feed.xml', c).encode('utf-8')
        fpath = '%s' % ('rss/',)
        self.write_file(fpath, t, 'xml')

class BuildPages(Build):
    def build(self):
        model = apps.get_model('pages', 'page')
        pages = model.objects.all()
        for page in pages:
            c = Context({'object':page,'SITE_URL':settings.SITE_URL, 'MEDIA_URL':settings.BAKED_MEDIA_URL})
            t = render_to_string(["details/%s.html" % page.slug, 'details/page.html'],c).encode('utf-8') 
            s = render_to_string('details/page.txt',c).encode('utf-8') 
            fpath = '%s' %(page.slug)
            self.write_file('', t, 'html', page.slug)
            self.write_file('', t, 'txt', page.slug)

class BuildMap(Build):
    def build(self):
        qs = apps.get_model('jrnl', 'entry').objects.filter(status__exact=1)
        cl = apps.get_model('locations', 'Country').objects.filter(visited=True).exclude(name='default')
        rl = apps.get_model('locations', 'Region').objects.all()
        rtl = apps.get_model('locations', 'Route').objects.all()
        c = Context({
            'object_list': qs,
            'country_list': cl,
            'region_list': rl,
            'route_list': rtl,
            'MEDIA_URL': settings.BAKED_MEDIA_URL,
            'IMAGES_URL': settings.BAKED_IMAGES_URL
        })
        t = render_to_string('archives/map_data.html', c).encode('utf-8')
        fpath = '%sdesign/templates/js/leaflet-providers.js' % settings.PROJ_ROOT
        leaflet_providers_js = open(fpath, 'r').read()
        js = leaflet_providers_js + t.decode(encoding='utf-8')
        self.write_file('media/js/', js.encode('utf-8'), 'js', 'mainmap')
        c = Context({
            'country_list': cl,
            'region_list': rl,
            'route_list': rtl,
            'MEDIA_URL': settings.BAKED_MEDIA_URL,
            'IMAGES_URL': settings.BAKED_IMAGES_URL
        })
        t = render_to_string('archives/map.html', c).encode('utf-8')
        self.write_file('', t, "html",'map')


# Back up entries to markdown text files which are then stored in dropbox and git 
class EntryBak(Build):
    def get_model_querset(self):
        model = apps.get_model('jrnl', 'entry')
        qs = model.objects.filter(status__exact=1)
        return qs

    def write_file(self, path, text_object):
        file = open(path, 'wb')
        file.write(text_object)
        file.close()

    def build_writing_bak(self):
        qs = self.get_model_querset()
        for obj in qs:
            c = Context({'object': obj, 'MEDIA_URL': settings.BAKED_MEDIA_URL, 'IMAGES_URL': settings.BAKED_IMAGES_URL})
            path = "%szbak/posts/%s_%s.txt" %(settings.PROJ_ROOT, (obj.pub_date.strftime("%Y-%m-%d").lower()), obj.slug)
            t = render_to_string('details/entry-bak.txt', c).encode('utf-8')
            self.write_file(path, t)



class BuildBooks(Build):
    def build(self):
        self.build_detail_pages()
        self.build_book_archive_pages()


    def build_book_archive_pages(self):
        qs = apps.get_model('books', 'Book').objects.all().order_by('-read_date').select_related()
        print(qs)
        self.build_archive_pages(qs, 'books/', 18)


    def build_detail_pages(self):
        qs = apps.get_model('books', 'Book').objects.all().order_by('-read_date').select_related()
        for book in qs:
            c = Context({'object': book,})
            t = render_to_string('details/book.html', c).encode('utf-8')
            path = 'books/'
            slug = '%s' % (book.slug)
            self.write_file(path, t, 'html', slug)