# python script to parte articles from Detroit, Chicago and Dallas news feeds
# must first install newspaper
# pip install newspaper#
# pip3 install newspaper3k                          

# uset wget get to download the image associated with the story
#wget -O image-file.jpg "https://images.foxtv.com/static.fox2detroit.com/www.fox2detroit.com/content/uploads/2020/11/1280/720/wjbk-vehicles-stolen-112320.jpg?ve=1\u0026tl=1"

#pass in the url link to the story
# e.g. "https://www.fox2detroit.com/news/detroit-police-investigating-multi-car-theft-at-east-side-dealership"


# start exection of script
#parse-detroit-articles.py ""https://www.fox2detroit.com/news/detroit-police-investigating-multi-car-theft-at-east-side-dealership"" "output-file.caption"

import sys
import os
from newspaper import Article

CITY=sys.argv[6]
tagCITY=CITY
print("tagged CITY=",tagCITY)

title=sys.argv[4]
print("\n")
print("BFORE Title=:",title)
title=title.replace('Γ','')
title=title.replace('Ç','\'')
title=title.replace('Ö','')
title=title.replace('ö','')
title=title.replace('ÿ','')
title=title.replace('╬','')
title=title.replace('ô','')
title=title.replace('├','')
title=title.replace('û','')
title=title.replace('┐','')
title=title.replace('-','-')
title=title.replace('∩','')
title=title.replace('╜','')
title=title.replace('ÿ','')
title=title.replace('|','')
title=title.replace('–','-')
title=title.replace('🚨',' ')
title=title.replace('‘',' ')
title=title.replace('|','-')
title=title.replace('£','')
title=title.replace('î','')
title=title.replace('É','e')
title=title.replace('╢','')
title=title.replace('┬','')
title=title.replace('ú','')
title=title.replace('«','')
title=title.replace('ë','e')
title=title.upper()                    
print("AFTER TitleConversion=:",title)

with open("igtitle.txt", "w", encoding="utf-8") as ttlfile: 
 ttlfile.write(title)
 ttlfile.close()
igpath=sys.argv[5]
print("\n")
print("py-ig-path=:",igpath)

url=sys.argv[1]
print("\n")
print("py-parse-news-article:",url)
print("\n")

#get output filename
fname=sys.argv[2]
fnpath=igpath+"\\"+fname+".txt"
print("py-output-filename:",fnpath)
print("\n")

# create file with title-in filename use first 30 characters of the string and remove special characters

igtitle=sys.argv[4]
fntitle=igtitle[1:50]
fntitle=fntitle.replace('Γ','')
fntitle=fntitle.replace('Ç','\'')
fntitle=fntitle.replace('Ö','')
fntitle=fntitle.replace('ö','')
fntitle=fntitle.replace('ÿ','')
fntitle=fntitle.replace('╬','')
fntitle=fntitle.replace('ô','')
fntitle=fntitle.replace('├','')
fntitle=fntitle.replace('û','')
fntitle=fntitle.replace('┐','')
fntitle=fntitle.replace('-','-')
fntitle=fntitle.replace('∩','')
fntitle=fntitle.replace('╜','')
fntitle=fntitle.replace('ÿ','')
fntitle=fntitle.replace('|','')
fntitle=fntitle.replace('–','-')
fntitle=fntitle.replace('🚨',' ')
fntitle=fntitle.replace('‘',' ')
fntitle=fntitle.replace('|','-')
fntitle=fntitle.replace('£','')
fntitle=fntitle.replace('î','')
fntitle=fntitle.replace('É','e')
fntitle=fntitle.replace('╢','')
fntitle=fntitle.replace('┬','')
fntitle=fntitle.replace('ú','')
fntitle=fntitle.replace('«','')
fntitle=fntitle.replace('ë','e')
fntitle=fntitle.replace(':','-')
fntitle=fntitle.replace(' ','-')
fntitle=fntitle.replace('&','-')
fntitle=fntitle.replace(',','-')
fntitle=fntitle.replace('\'','-')
fntitle=fntitle.replace('?','-')
fntitle=fntitle.replace('"','-')
tfpath=igpath+"\\"+fname+"-"+fntitle+".txt"
print("py-new-output-filename:",tfpath)
print("\n")

with open(tfpath, "w+", encoding="utf-8") as tfile:
 

 
 #download and parse article
 article=Article(url)
 article.download()
 article.parse()
 article.summary
 
 #print text of article to a file
 #limit article to 2200 characters
 #tfile.write(article.text[0:2125])
 modifiedarticle=article.text[0:2125]
 modifiedarticle=modifiedarticle.replace('—','-')
 modifiedarticle=modifiedarticle.replace('’','\'')
 modifiedarticle=modifiedarticle.replace('′','\'')
 modifiedarticle=modifiedarticle.replace('“','"')
 modifiedarticle=modifiedarticle.replace('”','"')
 modifiedarticle=modifiedarticle.replace('©',' ')
 modifiedarticle=modifiedarticle.replace('�',' ')
 modifiedarticle=modifiedarticle.replace('▼',' ')
 modifiedarticle=modifiedarticle.replace('ù',' ')
 modifiedarticle=modifiedarticle.replace('🙏',' ')
 modifiedarticle=modifiedarticle.replace('æ','\'')
 modifiedarticle=modifiedarticle.replace('û','-')
 modifiedarticle=modifiedarticle.replace('à',' ')
 modifiedarticle=modifiedarticle.replace('φ',' ')
 modifiedarticle=modifiedarticle.replace('±',' ')
 modifiedarticle=modifiedarticle.replace('≤',' ')
 modifiedarticle=modifiedarticle.replace('–','-')
 modifiedarticle=modifiedarticle.replace('🚨',' ')
 modifiedarticle=modifiedarticle.replace('‘',' ')
 modifiedarticle=modifiedarticle.replace('″','"')     
 tfile.write(modifiedarticle[0:2125])
                                                                         
 #add hashtags to end of file
 newssrc=sys.argv[3]
 tfile.write("")
 if tagCITY == "detroit":
   tfile.write("\r\n"+"#"+newssrc+"   #CrimeInTheD ™")
 else:
   tfile.write("\r\n"+"#"+newssrc+"   #crimewatch"+tagCITY)
 tfile.write("")
 
 tfile.close()
#the end
#print("\nMODIFIED ARTICLE BEFORE REMOVING TAGS")
#print(modifiedarticle)
#print("\n")

# remove unwanted tags from article summary file
with open(tfpath, "r+") as tfile:
 finalsummary=''
 for line in tfile:
  if "article" in line:
     print("\r\n-SKIPPED-"+line)
  elif "Next up in 5" in line:
     print("\r\n-SKIPPED-"+line)
  elif "Example video title" in line:
     print("\r\n-SKIPPED-"+line)   
  elif "[DOWNLOAD:" in line:
     print("\r\n-SKIPPED-"+line)
  elif "TRENDING STORIES:" in line:
     print("\r\n-SKIPPED-"+line)
  elif "[SIGN UP:" in line:
     print("\r\n-SKIPPED-"+line)   
  elif "RELATED:" in line:
     print("\r\n-SKIPPED-"+line)
  elif "Advertisement" in line:
     print("\r\n-SKIPPED-"+line)   
  elif "Copyright" in line:
     print("\r\n-SKIPPED-"+line)
  else:
     finalsummary=finalsummary+line    
 tfile.write(finalsummary)
 tfile.close()
#the end

# write final summary without tags to file 
with open(tfpath, "w") as summary: 
 summary.write(finalsummary)
 summary.close()
 
print("\nFINAL SUMMARY")
print(finalsummary)
print("\n")