import sys, io, re, json, glob, os
import urllib.request
from datetime import datetime, timedelta, date, time
import time as ttime
from bs4 import BeautifulSoup, SoupStrainer
from xml.sax.saxutils import escape
from collections import OrderedDict
import traceback
################################################################################
appname = 'RakiCal'
appvers = '2.10'
launch_duration = 45
refetch_back = 1
################################################################################
NL = "\n"
jours = ('lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi', 'dimanche')
repas = ('dejeuner', 'diner')
plats = ('entree_chaude', 'plat1', 'accompagnement1', 'plat2', 'accompagnement2', 'plat3', 'accompagnement3', 'dessert_chaud')
out_prefix = 'menu'
archives_path = './archives/'
base_url = 'http://services.telecom-bretagne.eu/rak/pagemenu.php'
semaines_to_parse = []
semaines_parsed = []
my_tags = SoupStrainer(['input','a','h2'])
date_mask = re.compile('([0-9]+)/([0-9]+)/([0-9]+)')
def date_to_timestamp(date):
#return int(datetime.combine(date,time()).timestamp())
return int(ttime.mktime(date.timetuple()))
# https://stackoverflow.com/questions/304256/whats-the-best-way-to-find-the-inverse-of-datetime-isocalendar
def iso_to_gregorian(iso_year, iso_week, iso_day):
# Gregorian calendar date for the given ISO year, week and day
fifth_jan = date(iso_year, 1, 5)
_, fifth_jan_week, fifth_jan_day = fifth_jan.isocalendar()
return fifth_jan + timedelta(days=iso_day-fifth_jan_day, weeks=iso_week-fifth_jan_week)
################################################################################
# => (date,menu)
def parse_page(date_to_parse=None):
if date_to_parse != None :
args = '?semaine='+str(date_to_timestamp(date_to_parse))
print('Parsing semaine '+str(date_to_parse)+' : ', file=sys.stderr, end='')
else:
args = ''
print('Parsing de la semaine courante : ',file=sys.stderr, end='')
try:
response = urllib.request.urlopen(base_url+args)
data = response.read()
charset = response.info().get_param('charset', 'UTF-8')
html = data.decode(charset)
#html = re.sub(' xmlns="[^"]+"', '', html, count=1) # sinon faut se taper le namespace dans le XPath
# Parse le HTML
soup = BeautifulSoup(html, 'html.parser', parse_only=my_tags)
# Extrait la date
str_date = soup.find('h2', {'id': 'lundi'}).text
match_date = re.search(date_mask, str_date)
week_date = date(int(match_date.group(3)), int(match_date.group(2)), int(match_date.group(1)))
semaines_parsed.append(week_date)
# Ajoute à la liste des semaines dispos celles trouvées si pas deja parsé
for semaine in soup.find_all('input', attrs={'name': 'semaine'}):
to_add = date.fromtimestamp(int(semaine['value']))
if to_add not in semaines_parsed:
semaines_to_parse.append(to_add)
# Extrait le menu
semaine = dict.fromkeys(jours)
for j in jours:
semaine[j] = dict.fromkeys(repas)
for r in repas:
semaine[j][r] = dict.fromkeys(plats)
for np,p in enumerate(plats):
try:
semaine[j][r][p] = soup('a', {'href': '#rampe_'+j+'_'+r+'_'+p}, limit=2)[1].text
except:
semaine[j][r][p] = None
# Met en cache la récup comme elle semble ok
isoc = week_date.isocalendar()
filename = archives_path+'menu_'+str(isoc[0])+'-'+str(isoc[1])+'.json'
with io.open(filename, 'w', encoding='utf8') as json_file:
json.dump(semaine, json_file, ensure_ascii=False)
print('ok', file=sys.stderr)
return (week_date,semaine)
except:
print('nok', file=sys.stderr)
traceback.print_stack()
return None
def get_from_cache(date_to_get):
print('Decaching semaine '+str(date_to_get)+' : ', file=sys.stderr, end='')
isoc = date_to_get.isocalendar()
filename = archives_path+'menu_'+str(isoc[0])+'-'+str(isoc[1])+'.json'
if (os.path.isfile(filename)):
try:
with open(filename, 'r', encoding='utf8') as f:
menu = json.load(f)
semaines_parsed.append(date_to_get)
print('ok', file=sys.stderr)
return (date_to_get, menu)
except:
print('nok', file=sys.stderr)
return None
else:
print('nok', file=sys.stderr)
return None
########################################
# Récupère et converti un menu en iCal
########################################
# https://drupal.org/node/61830
# http://www.ietf.org/rfc/rfc2445.txt
def iescape(s):
return s.replace('\\','\\\\').replace('\n','\\n').replace(',','\\,').replace(';','\\;')
def menu2iCal(date,menu):
iCal = ""
for d,day in enumerate(menu):
for r,repas in enumerate(day):
today = date+timedelta(days=d)
iCal += 'BEGIN:VEVENT'+NL
iCal += 'UID:'+appname+'_'+str(today)+'#'+str(r)+NL
#out("DTSTAMP:"+gmdate('Ymd')+'T'+gmdate('His'))
# iCal += 'DTSTAMP;TZID=Europe/Paris:'+datetime.now().strftime("%Y%m%dT%H%M%S")+NL
iCal += 'DTSTAMP:'+datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")+NL
iCal += 'SUMMARY:Repas au RAK'+NL
if r == 0:
iCal += 'DESCRIPTION:'+iescape(" Entrée : \n"+menu[d][r][0]+"\n\n Plat : \n"+menu[d][r][1]+"\n\n Dessert : \n"+menu[d][r][2])+NL
start = datetime.combine(today, time(12, 00))
else:
if menu[d][r][0]:
iCal += 'DESCRIPTION:'+iescape(" Entrée : \n"+menu[d][r][0]+"\n\n Plat : \n"+menu[d][r][1])+NL
else:
iCal += 'DESCRIPTION:'+iescape(" Plat : \n"+menu[d][r][1])+NL
start = datetime.combine(today, time(19, 15))
iCal += 'DTSTART;TZID=Europe/Paris:'+start.strftime("%Y%m%dT%H%M%S")+NL
#iCal += 'DTEND;TZID=Europe/Paris:'+(start+timedelta(minutes=launch_duration)).strftime("%Y%m%dT%H%M%S")+NL
iCal += 'DURATION:PT'+str(launch_duration)+'M'+NL
iCal += 'END:VEVENT'+NL
return iCal
########################################
# Génère l'iCal complet en générant chaque sous-iCal
########################################
def geniCal(menus):
iCal = 'BEGIN:VCALENDAR'+NL
iCal += 'VERSION:2.0'+NL
iCal += 'PRODID:-//ResEl (Alexandre Levavasseur)//'+appname+'/'+appvers+'//FR'+NL
iCal += 'CALSCALE:GREGORIAN'+NL
# Google, http://tzurl.org/zoneinfo/Europe/Paris
iCal += 'BEGIN:VTIMEZONE'+NL
iCal += 'TZID:Europe/Paris'+NL
iCal += 'X-LIC-LOCATION:Europe/Paris'+NL
iCal += 'BEGIN:DAYLIGHT'+NL
iCal += 'TZOFFSETFROM:+0100'+NL
iCal += 'TZOFFSETTO:+0200'+NL
iCal += 'TZNAME:CEST'+NL
iCal += 'DTSTART:19700329T020000'+NL
iCal += 'RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU'+NL
iCal += 'END:DAYLIGHT'+NL
iCal += 'BEGIN:STANDARD'+NL
iCal += 'TZOFFSETFROM:+0200'+NL
iCal += 'TZOFFSETTO:+0100'+NL
iCal += 'TZNAME:CET'+NL
iCal += 'DTSTART:19701025T030000'+NL
iCal += 'RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU'+NL
iCal += 'END:STANDARD'+NL
iCal += 'END:VTIMEZONE'+NL
for (date, menu) in menus.items():
iCal += menu2iCal(date,menu)
iCal += 'END:VCALENDAR'+NL
return iCal
########################################
# Récupère et converti un menu en jCal
########################################
# http://tools.ietf.org/html/rfc7265
# http://tools.ietf.org/html/rfc4627
def jescape(s):
return s.replace('\\','\\\\').replace('\n','\\n').replace('"','\\"').replace('/','\\/')
def menu2jCal(date,menu):
jCal = ""
for d,day in enumerate(menu):
for r,repas in enumerate(day):
today = date+timedelta(days=d)
jCal += ','+NL
jCal += '["vevent", ['+NL
jCal += ' ["uid", {}, "text", "'+appname+'_'+str(today)+'#'+str(r)+'"],'+NL
# jCal += ' ["dtstamp", {"tzid": "Europe/Paris"}, "date-time", "'+datetime.now().strftime("%Y-%m-%dT%H:%M:%S")+'"],'+NL
jCal += ' ["dtstamp", {}, "date-time", "'+datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")+'"],'+NL
jCal += ' ["summary", {}, "text", "Repas au RAK"],'+NL
if r == 0:
desc = ' Entrée : \n'+menu[d][r][0]+'\n\n Plat : \n'+menu[d][r][1]+'\n\n Dessert : \n'+menu[d][r][2]
start = datetime.combine(today, time(12, 00))
else:
if menu[d][r][0]:
desc = ' Entrée : \n'+menu[d][r][0]+'\n\n Plat : \n'+menu[d][r][1]
else:
desc = ' Plat : \n'+menu[d][r][1]
start = datetime.combine(today, time(19, 15))
jCal += ' ["description", {}, "text", "'+jescape(desc)+'"],'+NL
jCal += ' ["dtstart", {"tzid": "Europe/Paris"}, "date-time", "'+start.strftime("%Y-%m-%dT%H:%M:%S")+'"],'+NL
#jCal += ' ["dtend", {"tzid": "Europe/Paris"}, "date-time", "'+(start+timedelta(minutes=launch_duration)).strftime("%Y-%m-%dT%H:%M:%S")+'"],'+NL
jCal += ' ["duration", {}, "duration", "PT'+str(launch_duration)+'M"]'+NL
jCal += '], []]'
return jCal
########################################
# Génère le jCal complet en générant chaque sous-jCal
########################################
def genjCal(menus):
jCal = '["vcalendar",['+NL
jCal += ' ["version", {}, "text", "2.0"],'+NL
jCal += ' ["prodid", {}, "text", "-//ResEl (Alexandre Levavasseur)//'+appname+'/'+appvers+'//FR"],'+NL
jCal += ' ["calscale", {}, "text", "GREGORIAN"]'+NL
jCal += '],['+NL
# Google, http://tzurl.org/zoneinfo/Europe/Paris
jCal += '["vtimezone", ['+NL
jCal += ' ["tzid", {}, "text", "Europe/Paris"],'+NL
jCal += ' ["x-lic-location", {}, "text", "Europe/Paris"]'+NL
jCal += ' ],['+NL
jCal += ' ["daylight", ['+NL
jCal += ' ["tzoffsetfrom", {}, "utc-offset", "+01:00"],'+NL
jCal += ' ["tzoffsetto", {}, "utc-offset", "+02:00"],'+NL
jCal += ' ["tzname", {}, "text", "CEST"],'+NL
jCal += ' ["dtstart", {}, "date-time", "1970-03-29T02:00:00"],'+NL
jCal += ' ["rrule", {}, "recur", { "freq": "YEARLY", "bymonth": 3, "byday": "-1SU"}]'+NL
jCal += ' ], []'+NL
jCal += ' ], '+NL
jCal += ' ["standard", ['+NL
jCal += ' ["tzoffsetfrom", {}, "utc-offset", "+02:00"],'+NL
jCal += ' ["tzoffsetto", {}, "utc-offset", "+01:00"],'+NL
jCal += ' ["tzname", {}, "text", "CET"],'+NL
jCal += ' ["dtstart", {}, "date-time", "1970-10-25T03:00:00"],'+NL
jCal += ' ["rrule", {}, "recur", { "freq": "YEARLY", "bymonth": 10, "byday": "-1SU"}]'+NL
jCal += ' ], []'+NL
jCal += ' ] '+NL
jCal += ' ]'+NL
jCal += ']'
for (date, menu) in menus.items():
jCal += menu2jCal(date,menu)
jCal += ']]'+NL
return jCal
########################################
# Génère un XML contenant les menus
########################################
def genxml(menus):
xml = ''+NL
xml += ''+NL
for (date, menu) in menus.items():
for d,day in enumerate(menu):
xml += ' '+NL
xml += ''+NL
return xml
########################################
# Génère un JSON contenant les menus
########################################
def genjson(menus):
first = True
json = '{'+NL
for (date, menu) in menus.items():
for d,day in enumerate(menu):
if first:
first = False
else:
json += ','+NL
json += '"'+str(date+timedelta(days=d))+'":{'+NL
json += ' "lunch":{'+NL
json += ' "starter":"'+jescape(day[0][0])+'",'+NL
json += ' "main":"'+jescape(day[0][1])+'",'+NL
json += ' "dessert":"'+jescape(day[0][2])+'"'+NL
json += ' },'+NL
json += ' "dinner":{'+NL
if day[1][0]:
json += ' "starter":"'+jescape(day[1][0])+'",'+NL
json += ' "main":"'+jescape(day[1][1])+'"'+NL
json += ' }'+NL
json += '}'
json += NL+'}'+NL
return json
# semaine(jours{repas{plat}}) => menu(#jour[#repas[#plat]])
def genOldMenu(semaine):
menu = [[["","",""],["",""]] for i in range (7)]
for j,jour in enumerate(jours):
for r,repa in enumerate(repas):
now = semaine[jour][repa]
menu[j][r][0] = now['entree_chaude'] if now['entree_chaude'] else ''
menu[j][r][1] = '\n'.join(filter(None,(
', '.join(filter(None, (now['plat1'], now['accompagnement1']))),
', '.join(filter(None, (now['plat2'], now['accompagnement2']))),
', '.join(filter(None, (now['plat3'], now['accompagnement3'])))
)))
if (repa == 'dejeuner'):
menu[j][r][2] = now['dessert_chaud'] if now['dessert_chaud'] else ''
return menu
# -- Fetch les menus
# https://stackoverflow.com/questions/5280178/how-do-i-load-a-file-into-the-python-console
# exec(open('/tmp/test.py').read())
menus = {}
(first_week_date, menu) = parse_page() # Parse la semaine actuelle et va ajouter les semaines "autour"
menus[first_week_date] = menu # Ajoute aux menus
# Ajoute tout le cache aux fichiers à parcourir, les `refetch_back` seront refetchés
cache = sorted(glob.glob(archives_path+'menu_*.json'), key=os.path.basename, reverse=True)
file_mask = re.compile('menu_(?P[0-9]+)-(?P[0-9]+)\.json')
for file in cache[:78]:
res = re.search(file_mask, file)
week = iso_to_gregorian(int(res.group('year')), int(res.group('week')), 1)
if (week not in semaines_to_parse) and (week not in semaines_parsed):
semaines_to_parse.append(week)
# Va fetcher !
semaines_to_parse2 = semaines_to_parse[:] # Hack pour arrêter de récupérer des semaines
for semaine in semaines_to_parse2: # Le pointeur va bien jusqu'à la fin de la liste si elle est mise à jour en cours de route
if semaine in menus:
continue
try:
if (semaine >= first_week_date - timedelta(weeks=refetch_back)): # Refetch jusqu'à `refetch_back` semaines
res = parse_page(semaine)
if res:
(week_date, menu) = res
menus[week_date] = menu
else: # Fallback sur le cache si il existe
(week_date, menu) = get_from_cache(semaine)
if menu:
menus[week_date] = menu
else: #Sinon on utilise le cache
(week_date, menu) = get_from_cache(semaine)
if menu:
menus[week_date] = menu
else: # Fallback sur un fetching
res = parse_page(semaine)
if res:
(week_date, menu) = res
menus[week_date] = menu
except:
raise
# Converti les menus dans "l'ancien format"
oldMenus = {}
for (week_date,menu) in menus.items():
oldMenus[week_date] = genOldMenu(menus[week_date]) # Conversion
# Trie par date décroissante
oldMenus = OrderedDict(sorted(oldMenus.items(), reverse=True))
# Génère les menus dans les formats voulus
print ('Génération du format ics : '+out_prefix+'.ics', file=sys.stderr)
ical_data = geniCal(oldMenus)
ical_file = open(out_prefix+'.ics', 'w', encoding="UTF-8", newline='\r\n')
ical_file.write(ical_data)
ical_file.close()
print ('Génération du format jcs : '+out_prefix+'.jcs', file=sys.stderr)
jcal_data = genjCal(oldMenus)
jcal_file = open(out_prefix+'.jcs', 'w', encoding="UTF-8", newline='\r\n')
jcal_file.write(jcal_data)
jcal_file.close()
print ('Génération du format xml : '+out_prefix+'.xml', file=sys.stderr)
xml_data = genxml(oldMenus)
xml_file = open(out_prefix+'.xml', 'w', encoding="UTF-8", newline='\r\n')
xml_file.write(xml_data)
xml_file.close()
print ('Génération du format json : '+out_prefix+'.json', file=sys.stderr)
json_data = genjson(oldMenus)
json_file = open(out_prefix+'.json', 'w', encoding="UTF-8", newline='\r\n')
json_file.write(json_data)
json_file.close()
sys.exit(0)