#!/usr/bin/env python """ Script that does some processing for the automatic update of the ECMWF forecast web page. The script performs the following actions: Loop over all forecast directories specified: 1) Determine which subdirectories intended for ECMWF images are present. 2) Split the retrieved directory names into empty and full directories. 3) Determine the newest forecasts (number can be specified) and write a new 'settings.php' for the web page. 4) Delete old empty subdirectories that are not needed anylonger. 5) Archive old full subdirectories, then delete them from the web server. 6) Create new subdirectories for future incoming plots (if they are not filled lateron they will be automatically deleted in a later run of this script). To use this script, a) Set up all the web pages (no subdirectories for the plots required, they will be created from this script). b) Create templates for 'settings.php' (only the '$init_times' line will will modified). c) Create directories for the archives (see 'archiveBaseDir' below). d) Adjust the settings below. e) Insert this script as a cronjob to your crontab: # # cronjob for the polarcat campaign. checks for new foerecasts, # updates the webpage and the directory structure. runs every # six minutes. 0-59/6 * * * * /home/http/docs/polarcat/restricted/scripts/update_ecmwf_webpage.py >> /home/http/docs/polarcat/restricted/scripts/update.log 2>&1 That's it! :-) Marc Rautenhaus, June 2008 :: marc.rautenhaus@dlr.de """ import os import datetime import stat import tarfile import shutil import socket import copy # ################################################################################ # # SETTINGS # ################################################################################ # # The name of the campagin -- only for print output. campaignName = "dlr/ipa missionsupport classic webpage" # # Directory settings for all forecast directories that should be processed. # 'frecastBases' is a list of tuples containing three elements each: # [(forecastBaseDir, archiveBaseDir, settingsTemplate), (...), ...] # 'forecastBaseDir' is the web page base directory for the current forecast # type. All web page files are located here, the 'setting.php' file # will be stored here; the ECMWF plots will be stored in subdirectories # of format YYYYMMDDHHMM of this directory. # 'archiveBaseDir' is the directory to which old forecasts from the # forecastBaseDir will be archived. Archived directories will have the # name YYYYMMDDHHMM.tar.gz. # 'settingsTemplate' gives the complete path to a template of 'settings.php' # for the forecast web page. This file will be copied to # forecastBaseDir/settings.php with the line containing '$init_times' # replaced. # forecastBases can contain as many tuples as you like, all specified # directories will be processed. forecastBases = [("/var/www/html/forecasts", "/var/www/html/scripts/settings_template_forecasts.php") ] # # The number of ECMWF init times selectable in the drop down box on the web page. numberSelectableForecasts = 6 # # Number of directories that should be created for future incoming forecasts. # The counter starts at 'lastECMWFInit - deleteEmptyDirsOlderThanDays'. createDirsNumberRunsInAdvance = 10 # # Delete forecast plot directories with a data older than # 'deleteEmptyDirsOlderThanDays' with respect to the last ECMWF init time. deleteEmptyDirsOlderThanDays = 1 # # Name convention that the subdirectories containing the ECMWF plots follow. # Compare to 'datetime' documentation on the Python homepage. timeFormat = "%Y%m%d%H%M" # YYYYMMDDhhmm # # Files in the forecast directories can be renamed, for example, if suffixes # indicating model levels should be changed to suffixes indicating the # correspondig approximate pressure levels. If you wish to rename files, # set 'renameFiles' to True and place the strings that should be renamed # into the dictionary 'filenameReplacementTable': In all filenames that contain # a key the key will be changed to the corresponding value. # NOTE: Renaming files can take some time -- only use it when you need to. renameFiles = False filenameReplacementTable = {'_52_': '_200_', '_55_': '_250_', '_60_': '_300_', '_64_': '_500_', '_71_': '_700_', '_77_': '_850_', '_81_': '_925_', '_88_': '_1000_'} # ################################################################################ ################################################################################ # MAIN PROGRAM ################################################################################ print "*****************************************************************" print "ecmwf forecasts web page processing for", campaignName print "starting at " + datetime.datetime.ctime(datetime.datetime.now()) + \ " on " + socket.gethostname() # # Get current time and compute the time the last ECMWF run was started # (last 00h or 12h time). currentUTCTime = datetime.datetime.utcnow() lastECMWFInitHour = 0 if currentUTCTime.hour < 12 else 12 lastECMWFInit = currentUTCTime.replace(hour = lastECMWFInitHour, minute = 0) print "\ncurrent time is", currentUTCTime.strftime(timeFormat), \ "utc; last ECMWF run was started on", lastECMWFInit.strftime(timeFormat) # # Loop over all forecast directories to be processed. for forecastBaseDir, archiveBaseDir, settingsTemplate in forecastBases: print "\n\nprocessing base directory is", forecastBaseDir # # Get a list of all subdirectories in the base dir path. baseDirEntries = os.listdir(forecastBaseDir) forecastDirs = [fcDir for fcDir in baseDirEntries \ if os.path.isdir(os.path.join(forecastBaseDir, fcDir))] # # Test if all directory names in 'forecastDirs' follow the name conventions # and can be converted to a time. If the conversion fails, it's not a # directory that contains forecast data and should be removed from the list. for fcDir in copy.deepcopy(forecastDirs): try: tmp = datetime.datetime.strptime(fcDir, timeFormat) except: forecastDirs.remove(fcDir) # # Filter those subdirectories that are empty (i.e. contain no forecast # files). emptyForecastDirs = [fcDir for fcDir in forecastDirs \ if len(os.listdir(os.path.join(forecastBaseDir, fcDir))) == 0] emptyForecastDirs.sort() print "\nbase directory contains as empty subdirectories:\n", emptyForecastDirs # # Filter those subdirectories that contain files. filledForecastDirs = [fcDir for fcDir in forecastDirs \ if len(os.listdir(os.path.join(forecastBaseDir, fcDir))) > 0] filledForecastDirs.sort(reverse = True) print "\nbase directory contains as filled subdirectories:\n", filledForecastDirs # # The newest 'numberSelectableForecasts' forecast subdirectories will be # written to 'settings.php', so that they can be selected as forecast runs # on the webpage. If less than 'numberSelectableForecasts' forecast runs # are available select as many runs as possible. if len(filledForecastDirs) > 0: selectableForecastDirs = filledForecastDirs[0:min(numberSelectableForecasts, len(filledForecastDirs))] print "\nforecast runs that will be selectable on the webpage (written to " \ "settings.php):\n", selectableForecastDirs # # Create the '$init_times' line to be written to setting.php. settings_php_line = " $init_times = array(" for forecastDir in selectableForecastDirs: settings_php_line += ('"%s"' % forecastDir) + ', ' settings_php_line = settings_php_line[0:len(settings_php_line)-2] + ");" # # Open the template settings file in read mode and 'setting.php' in # the forecast base directory in write mode; copy the entire file # except for the '$init_times' line, which is replaced by the above line. settingsTemplateFile = open(settingsTemplate, 'r') settingsFileName = os.path.join(forecastBaseDir, "settings.php") settingsFile = open(settingsFileName, 'w') print " Writing", settingsFileName for line in settingsTemplateFile.readlines(): if line.find("$init_times") > 0: settingsFile.write(settings_php_line) else: settingsFile.write(line) settingsTemplateFile.close() settingsFile.close() os.chmod(settingsFileName, stat.S_IRUSR | stat.S_IWUSR | \ stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) # # Determine which of the empty subdirectories are old enough so that no forecast # will arrive in them anymore: All empty directories with a date older than # the newest full forecast dir; if no full forecast dir exists all subdirs that # are older than 'deleteEmptyDirsOlderThanDays' if len(filledForecastDirs) > 0: emptyDirsToDelete = [fcDir for fcDir in emptyForecastDirs if fcDir < filledForecastDirs[0]] else: # if no filled forecast dir delete all empty dirs older than 'deleteEmptyDirsOlderThanDays' deleteDirsOlderThanTime = lastECMWFInit - datetime.timedelta(deleteEmptyDirsOlderThanDays) emptyDirsToDelete = [fcDir for fcDir in emptyForecastDirs if fcDir < deleteDirsOlderThanTime] print "\nempty subdirectories to be deleted:\n", emptyDirsToDelete for dirToDelete in emptyDirsToDelete: print " removing", dirToDelete try: os.rmdir(os.path.join(forecastBaseDir, dirToDelete)) except: print "\tan error occurred, skipping." # # If files in the forecast directories should be renamed, do so now: Search # for occurences of the keys contained in 'filenameReplacementTable' in # the filenames; if an occurence has been found replace the key with its # corresponding value and rename the file. if renameFiles: print "\nrenaming files." for directory in filledForecastDirs: for name in os.listdir(os.path.join(forecastBaseDir, directory)): for key, value in filenameReplacementTable.items(): if name.find(key) >= 0: shutil.move(os.path.join(forecastBaseDir, directory, name), os.path.join(forecastBaseDir, directory, name.replace(key, value))) # # All forecast directories older than the newest forecast minus # 'numberSelectableForecasts' are not needed anylonger. They can be # archived or simply deleted. filledDirsToArchive = filledForecastDirs[min(numberSelectableForecasts, len(filledForecastDirs)):len(filledForecastDirs)] print "\nold filled forecast subdirectories to be archived:\n", filledDirsToArchive # # Put files in directories to be archived into tar.gz archives located # at 'archiveBaseDir'. for dirToArchive in filledDirsToArchive: #sg archiveName = os.path.join(archiveBaseDir, dirToArchive + ".tar.gz") #sg print " Archiving", dirToArchive, "to", archiveName #sg if os.path.isfile(archiveName): #sg print "\tarchive exists, skipping." #sg continue #sg try: #sg tar = tarfile.open(archiveName, "w:gz") #sg for name in os.listdir(os.path.join(forecastBaseDir, dirToArchive)): #sg tar.add(os.path.join(forecastBaseDir, dirToArchive, name)) #sg tar.close() #sg except: #sg print "\tan error occurred, skipping." print " Deleting", dirToArchive try: shutil.rmtree(os.path.join(forecastBaseDir, dirToArchive)) except: print "\tan error occurred, skipping." # # Create a list of subdirectories to be created for future incoming forecasts: # 'createDirsNumberRunsInAdvance' directories starting from 'lastECMWFInit' - # 'deleteEmptyDirsOlderThanDays' will be created. createEmptyDirs = [] for run in range(createDirsNumberRunsInAdvance+1): createDirTime = lastECMWFInit \ - datetime.timedelta(deleteEmptyDirsOlderThanDays) \ + datetime.timedelta(0.5 * run) dirToCreate = createDirTime.strftime(timeFormat) createEmptyDirs.append(dirToCreate) print "\nnew subdirectories for incoming forecasts to be created:\n", createEmptyDirs # # Create directories, change rights so that all group members have write access. for createDir in createEmptyDirs: print " Creating directory", createDir try: os.mkdir(os.path.join(forecastBaseDir, createDir)) os.chmod(os.path.join(forecastBaseDir, createDir), stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) except OSError: print "\texists, skipping." print "\n\nprocessing finished at", datetime.datetime.ctime(datetime.datetime.now()) print "*****************************************************************"