updated links app to grab full text of page as markdown

7 files changed, 67 insertions, 4 deletions

diff --git a/app/links/admin.py b/app/links/admin.py
index 4a85e9a..ff75a09 100644
--- a/app/links/admin.py
+++ b/app/links/admin.py
@@ -14,6 +14,7 @@ class LinkAdmin(admin.ModelAdmin):
                 'title',
                 'url',
                 'description',
+                'body_markdown',
             )
         }),
         ('Details', {
diff --git a/app/links/migrations/0001_initial.py b/app/links/migrations/0001_initial.py
new file mode 100644
index 0000000..1a50e8e
--- /dev/null
+++ b/app/links/migrations/0001_initial.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9 on 2015-12-22 09:22
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('taggit', '0002_auto_20150616_2121'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Link',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=400)),
+                ('url', models.CharField(max_length=400)),
+                ('description', models.TextField(blank=True, null=True)),
+                ('pub_date', models.DateTimeField()),
+                ('status', models.IntegerField(choices=[(0, 'Private'), (1, 'Public')], default=0)),
+                ('tags', taggit.managers.TaggableManager(blank=True, help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags')),
+            ],
+            options={
+                'ordering': ['-pub_date'],
+            },
+        ),
+    ]
diff --git a/app/links/migrations/0002_link_body_markdown.py b/app/links/migrations/0002_link_body_markdown.py
new file mode 100644
index 0000000..5930a90
--- /dev/null
+++ b/app/links/migrations/0002_link_body_markdown.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9 on 2015-12-22 09:22
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('links', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='link',
+            name='body_markdown',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]
diff --git a/app/links/migrations/__init__.py b/app/links/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/app/links/migrations/__init__.py
diff --git a/app/links/models.py b/app/links/models.py
index d2b6ff7..a343604 100644
--- a/app/links/models.py
+++ b/app/links/models.py
@@ -13,6 +13,7 @@ class Link(models.Model):
     title = models.CharField(max_length=400)
     url = models.CharField(max_length=400)
     description = models.TextField(blank=True, null=True)
+    body_markdown = models.TextField(blank=True, null=True)
     pub_date = models.DateTimeField()
     PUB_STATUS = (
         (0, 'Private'),
diff --git a/app/links/retriever.py b/app/links/retriever.py
index 59b90ec..ec7f834 100644
--- a/app/links/retriever.py
+++ b/app/links/retriever.py
@@ -18,15 +18,17 @@ def sync_pinboard_links():
     for link in links:
         try:
             # check to see if link exists
-            row = Link.objects.get(link_id=link['hash'])
+            row = Link.objects.get(url=link['href'])
             print("already have" + row.title)
         except ObjectDoesNotExist:
+            md = get_markdown(link['href'])
             l, created = Link.objects.get_or_create(
                 title=link['description'],
                 url=link['href'],
                 description=link['extended'],
                 pub_date=datetime.datetime.strptime(link['time'], "%Y-%m-%dT%H:%M:%SZ"),
-                status=0
+                status=0,
+                body_markdown=md['markdown']
             )
             if created:
                 print(l.title)
@@ -35,6 +37,13 @@ def sync_pinboard_links():
             email_link(l)
 
 
+def get_markdown(source):
+    url = "http://heckyesmarkdown.com/go/?read=1&preview=0&showframe=0&output=json&u=%s" % (source)
+    r = requests.get(url, timeout=15.001)
+    data = r.json()
+    return data
+
+
 def email_link(link):
     """
     Sends an imported link to Gmail (never hurts to have backups)
diff --git a/app/links/sync_links.py b/app/links/sync_links.py
index a9fa1bf..f99b7c4 100644
--- a/app/links/sync_links.py
+++ b/app/links/sync_links.py
@@ -3,12 +3,11 @@ import os
 from os.path import dirname, abspath
 import django
 PROJECT_ROOT = abspath(dirname(dirname(dirname(__file__)))) + '/'
-#PROJECT_ROOT = abspath(dirname(dirname(__file__)))
 sys.path.append(PROJECT_ROOT)
 sys.path.append(PROJECT_ROOT + '/app')
 sys.path.append(PROJECT_ROOT + '/app/lib')
 sys.path.append(PROJECT_ROOT + '/config')
-sys.path.append('/home/luxagraf/apps/venv/bin/python2.7/')
+sys.path.append(PROJECT_ROOT + '/venv/bin/python3')
 os.environ['DJANGO_SETTINGS_MODULE'] = 'config.settings'
 django.setup()
 from links import retriever