updated links app to grab full text of page as markdown

1 files changed, 11 insertions, 2 deletions

diff --git a/app/links/retriever.py b/app/links/retriever.py
index 59b90ec..ec7f834 100644
--- a/app/links/retriever.py
+++ b/app/links/retriever.py
@@ -18,15 +18,17 @@ def sync_pinboard_links():
     for link in links:
         try:
             # check to see if link exists
-            row = Link.objects.get(link_id=link['hash'])
+            row = Link.objects.get(url=link['href'])
             print("already have" + row.title)
         except ObjectDoesNotExist:
+            md = get_markdown(link['href'])
             l, created = Link.objects.get_or_create(
                 title=link['description'],
                 url=link['href'],
                 description=link['extended'],
                 pub_date=datetime.datetime.strptime(link['time'], "%Y-%m-%dT%H:%M:%SZ"),
-                status=0
+                status=0,
+                body_markdown=md['markdown']
             )
             if created:
                 print(l.title)
@@ -35,6 +37,13 @@ def sync_pinboard_links():
             email_link(l)
 
 
+def get_markdown(source):
+    url = "http://heckyesmarkdown.com/go/?read=1&preview=0&showframe=0&output=json&u=%s" % (source)
+    r = requests.get(url, timeout=15.001)
+    data = r.json()
+    return data
+
+
 def email_link(link):
     """
     Sends an imported link to Gmail (never hurts to have backups)