improve deprecated code - add dates, check expire, improve parse of api webpage

Author: mahtin

CloudFlare/api_decode_from_web.py

@@ -1,5 +1,7 @@
 """ API extras for Cloudflare API"""
 
+import datetime
+
 from bs4 import BeautifulSoup, Comment
 
 API_TYPES = ['GET', 'POST', 'PATCH', 'PUT', 'DELETE']
@@ -10,9 +12,31 @@ def do_section(section):
     cmds = []
     # look for deprecated first in section
     deprecated = False
+    deprecated_date = ''
+    deprecated_already = False
     for tag2 in section.find_all('h3'):
+        # <h3 class="text-warning" data-reactid="13490">Deprecation Warning</h3>
         if 'Deprecation Warning' in str(tag2):
             deprecated = True
+            break
+    for tag2 in section.find_all('p'):
+        # <p class="deprecation-date" data-reactid="13491">End of life Date: November 2, 2020</p>
+        if 'End of life Date:' in str(tag2):
+            for child in tag2.children:
+                deprecated_date = str(child).replace('End of life Date:','').strip()
+                try:
+                    # clean up date
+                    d = datetime.datetime.strptime(deprecated_date, '%B %d, %Y')
+                    if d <= datetime.datetime.now():
+                        # already done!
+                        deprecated_already = True
+                    deprecated_date = d.strftime('%Y-%m-%d')
+                except ValueError:
+                    # Lets not worry about all the date formats that could show-up. Leave as a string
+                    pass
+                break
+        if deprecated_date != '':
+            break
     # look for all API calls in section
     for tag2 in section.find_all('pre'):
         cmd = []
@@ -29,7 +53,7 @@ def do_section(section):
         cmd = ''.join(cmd[1:])
         if cmd[0] == '/':
             cmd = cmd[1:]
-        v = {'deprecated': deprecated, 'action': action, 'cmd': cmd}
+        v = {'action': action, 'cmd': cmd, 'deprecated': deprecated, 'deprecated_date': deprecated_date, 'deprecated_already': deprecated_already}
         cmds.append(v)
     return cmds
 

Makefile

@@ -79,11 +79,14 @@ lint:
 api:
 	@tmp=/tmp/_$$$$_ ; \
 	python -m cli4 --dump | sed -e 's/^\///' | sort > $$tmp.1 ; \
-	python -m cli4 --api | sed -e 's/.* //' -e 's/\/:[^:\/]*//g' -e 's/^\///' | sort | uniq > $$tmp.2 ; \
+	python -m cli4 --api | sed -e 's/^[A-Z][A-Z]*  *//' -e 's/\/:[a-z_]*//g' -e 's/^\///' | sort | uniq > $$tmp.2 ; \
+	egrep -v '; deprecated' < $$tmp.2 | diff $$tmp.1 - > $$tmp.3 ; \
 	echo "In code:" ; \
-	diff $$tmp.1 $$tmp.2 | egrep '< ' | sed -e 's/< /         /' | sort ; \
+	egrep '< ' < $$tmp.3 | sed -e 's/< /         /' | sort | tee $$tmp.4 ; \
 	echo "In docs:" ; \
-	diff $$tmp.1 $$tmp.2 | egrep '> ' | sed -e 's/> /         /' | sort ; \
+	egrep '> ' < $$tmp.3 | sed -e 's/> /         /' | sort ; \
+	echo "Deprecated:" ; \
+        egrep '; deprecated' < $$tmp.2 | while read cmd x depricated depricated_date ; do egrep "$$cmd" $$tmp.4 | sed -e "s/$$/ ; depricated $$depricated_date/" ; done | sort | uniq ; \
 	rm $$tmp.?
 
 clean:

cli4/cli4.py

@@ -29,8 +29,12 @@ def dump_commands_from_web():
     w = cf.api_from_web()
     for r in w:
         if r['deprecated']:
-            continue
-        sys.stdout.write('%-6s %s\n' % (r['action'], r['cmd']))
+            if r['deprecated_already']:
+                sys.stdout.write('%-6s %s ; deprecated %s - expired!\n' % (r['action'], r['cmd'], r['deprecated_date']))
+            else:
+                sys.stdout.write('%-6s %s ; deprecated %s\n' % (r['action'], r['cmd'], r['deprecated_date']))
+        else:
+            sys.stdout.write('%-6s %s\n' % (r['action'], r['cmd']))
 
 def run_command(cf, method, command, params=None, content=None, files=None):
     """run the command line"""