""" Creation of resource descriptors The first part of this is an early experiment to automatically create resource descriptors from structured data representations. While parts of this may be recoverable for smarter gavo start functionality, doing this so that the result is actually useful is hard. Instead, the new gavo start functionality just fetches one of a few commented RD templates, fills out a thing or two and leaves the rest to the operator. """ #c Copyright 2008-2025, the GAVO project #c #c This program is free software, covered by the GNU GPL. See the #c COPYING file in the source distribution. import datetime import os import re import sys import pkg_resources from gavo import base from gavo import grammars from gavo import rscdef from gavo import votable from gavo import utils from gavo.base import macros from gavo.grammars import fitsprodgrammar from gavo.grammars import fitstablegrammar from gavo.formats import votableread from gavo.utils import ElementTree from gavo.utils import fitstools MS = base.makeStruct FT_TYPE_MAP = { "L": "boolean", "B": "bytea", "A": "char", "I": "smallint", "J": "integer", "K": "bigint", "E": "real", "D": "double precision", } def getTypeForFTFormat(format, name): """returns a DaCHS type for FITS table column. """ typeCode = None # variable-length array FITS codes mat = re.match(r"(\d*)P(.)\(\d*\)$", format) if mat: typeCode = mat.group(2) arrSize = "100" # just a sentinel # "conventional FITS codes else: mat = re.match(r"(\d*)(.)$", format) if mat: arrSize, typeCode = mat.group(1), mat.group(2) if typeCode not in FT_TYPE_MAP: raise base.ReportableError(f"FITS type code '{format}' of {name}" " not handled by gavo mkrd; add handling if you can.") dbType = FT_TYPE_MAP[typeCode] if arrSize and int(arrSize)>1: if dbType=="char": dbType = "text" else: dbType = dbType+"[]" return dbType # ======================= Begin deprecated ui and implementation =========== ignoredFITSHeaders = set(["COMMENT", "SIMPLE", "BITPIX", "EXTEND", "NEXTEND", "SOFTNAME", "SOFTVERS", "SOFTDATE", "SOFTAUTH", "SOFTINST", "HISTORY", "BZERO", "BSCALE", "DATAMIN", "DATAMAX"]) wcsKey = re.compile("CD.*|CRVAL.*|CDELT.*|NAXIS.*|CRPIX.*|CTYPE.*|CUNIT.*" r"|CROTA.*|RADECSYS|AP?_\d_\d|BP?_\d_\d|LATPOLE|LONPOLE") def isIgnoredKeyword(kw): """returns true if kw should not be translated or put into the table. This is important for all WCS keywords when you want to compute SIAP bboxes; these keywords must not be translated. """ return kw in ignoredFITSHeaders or wcsKey.match(kw) def makeTableFromFT(rd, srcName, opts): from gavo.utils import pyfits cols, nameMaps = [], {} nameMaker = base.VOTNameMaker() hdus = pyfits.open(srcName) for fitsCol in hdus[1].columns: destName = nameMaker.makeName(fitsCol) if destName!=fitsCol.name: nameMaps[destName] = fitsCol.name cols.append(MS(rscdef.Column, type=getTypeForFTFormat(fitsCol.format, fitsCol.name), name=destName, unit=fitsCol.unit, ucd="", description="FILL IN")) table = rscdef.TableDef(rd, id=opts.tableName, onDisk=True, columns=cols) table.nameMaps = nameMaps hdus.close() return table def makeDataForFT(rd, srcName, opts): targetTable = rd.tables[0] # nameMaps left by makeTableFromFT rmkMaps =[MS(rscdef.MapRule, key=key, content_="vars['%s']"%src) for key, src in targetTable.nameMaps.items()] grammar = MS(fitstablegrammar.FITSTableGrammar) sources = MS(rscdef.SourceSpec, pattern=["*.fits"], recurse=True) rowmaker = MS(rscdef.RowmakerDef, id="gen_rmk", idmaps="*", maps=[rmkMaps]) make = MS(rscdef.Make, table=targetTable, rowmaker=rowmaker) return MS(rscdef.DataDescriptor, id="import", sources=sources, grammar=grammar, rowmakers=[rowmaker], make=make) def makeTableFromFITS(rd, srcName, opts): keyMappings = [] table = rscdef.TableDef(rd, id=opts.tableName, onDisk=True) hdus = fitstools.openFits(srcName) headerCards = hdus[0].header.cards for index, card in enumerate(headerCards): if isIgnoredKeyword(card.keyword): continue colName = re.sub("[^a-z]", "_", card.keyword.lower()) if not colName: continue if isinstance(card.value, str): type = "text" elif isinstance(card.value, int): type = "integer" else: type = "real" table.feedObject("column", MS(rscdef.Column, name=colName, unit="FILLIN", ucd="FILLIN", type=type, description=card.comment)) keyMappings.append((colName, card.keyword)) rd.setProperty("mapKeys", ", ".join("%s:%s"%(v,k) for k,v in keyMappings)) hdus.close() return table.finishElement() def makeDataForFITS(rd, srcName, opts): targetTable = rd.tables[0] dd = rscdef.DataDescriptor(rd, id="import_"+opts.tableName) grammar = fitsprodgrammar.FITSProdGrammar(dd) grammar.feedObject("qnd", True) rowfilter = base.parseFromString(grammars.Rowfilter, """ "%s" "FILLIN" "FILLIN" """%(targetTable.getQName())) grammar.feedObject("rowfilter", rowfilter) grammar.feedObject("mapKeys", MS(grammars.MapKeys, content_=rd.getProperty("mapKeys"))) grammar.finishElement() dd.grammar = grammar dd.feedObject("sources", MS(rscdef.SourceSpec, pattern=["*.fits"], recurse=True)) dd.feedObject("rowmaker", MS(rscdef.RowmakerDef, idmaps="*", id="gen_rmk")) dd.feedObject("make", MS(rscdef.Make, table=targetTable, rowmaker="gen_rmk")) return dd def makeTableFromVOTable(rd, srcName, opts): with open(srcName, "rb") as f: rawTable = next(votable.parse(f)) return votableread.makeTableDefForVOTable(opts.tableName, rawTable.tableDefinition, onDisk=True) def makeDataForVOTable(rd, srcName, opts): rowmaker = MS(rscdef.RowmakerDef, id="makerows_"+opts.tableName, idmaps="*") # The qualifiedId monkeying is necessary since otherwise the # ReferenceAttribute.unparse thinks it's ok to return the objects raw. # Face it: I've not really had serialization in mind when writing all # this. rowmaker.qualifiedId = rowmaker.id rd.tables[0].qualifiedId = rd.tables[0].id return MS(rscdef.DataDescriptor, grammar=MS(rscdef.getGrammar("voTableGrammar")), sources=MS(rscdef.SourceSpec, pattern=srcName), rowmaker=rowmaker, makes=[MS(rscdef.Make, table=rd.tables[0], rowmaker=rowmaker)]) tableMakers = { "FITS": makeTableFromFITS, "VOT": makeTableFromVOTable, "FT": makeTableFromFT, } dataMakers = { "FITS": makeDataForFITS, "VOT": makeDataForVOTable, "FT": makeDataForFT, } def makeRD(args, opts): from gavo import rscdesc rd = rscdesc.RD(None, schema=os.path.basename(opts.resdir), resdir=opts.resdir) for key, value in [ ("title", "FILL-IN"), ("creationDate", utils.formatISODT(datetime.datetime.utcnow())), ("description", "FILL-IN a long text (and maybe do format='plain'" " or even format='rst'"), ("copyright", "Free to use."), ("creator.name", "Author, S."), ("creator", ""), ("creator.name", "Other, A."), ("subject", "One Keyword"), ("subject", "Two Words"), ("content.type", "Catalog"), ("coverage.waveband", "Optical"), ("coverage.profile", "AllSky ICRS"),]: rd.addMeta(key, value) rd.feedObject("table", tableMakers[opts.srcForm](rd, args[0], opts)) rd.feedObject("data", dataMakers[opts.srcForm](rd, args[0], opts)) return rd.finishElement() def indent(elem, level=0): i = "\n" + level*"\t" if len(elem): if not elem.text or not elem.text.strip(): elem.text = i + "\t" if not elem.tail or not elem.tail.strip(): elem.tail = i for child in elem: indent(child, level+1) if not child.tail or not child.tail.strip(): child.tail = i if not elem.tail or not elem.tail.strip(): elem.tail = i else: if level and (not elem.tail or not elem.tail.strip()): elem.tail = i def writePrettyPrintedXML(root): indent(root) ElementTree.ElementTree(root).write(sys.stdout.buffer, encoding="utf-8") def parseCommandLine(): from optparse import OptionParser parser = OptionParser(usage = "%prog [options] " " DEPRECATED, use dachs start instead") parser.add_option("-f", "--format", help="Input file format: " " FITS, VOT or FT (FITS table)" " Default: Detect from file name", dest="srcForm", default=None, action="store", type="str") parser.add_option("-t", "--table-name", help="Name of the generated table", dest="tableName", default="main", action="store", type="str") parser.add_option("-r", "--resdir", help="Override resdir (and schema)", dest="resdir", default=os.getcwd(), action="store", type="str") opts, args = parser.parse_args() if len(args)!=1: parser.print_help(file=sys.stderr) sys.exit(1) if not opts.srcForm: ext = os.path.splitext(args[0])[1].lower() if ext in set([".xml", ".vot"]): opts.srcForm = "VOT" elif ext==".fits": opts.srcForm = "FITS" else: sys.stderr.write("Cannot guess format, use -f option: %s\n"%args[0]) parser.print_help(file=sys.stderr) sys.exit(1) return opts, args def main(): # hack to make id and onDisk copyable so we see them on iterEvent rscdef.TableDef._id.copyable = rscdef.TableDef._onDisk.copyable = True rscdef.DataDescriptor._id.copyable = True opts, args = parseCommandLine() rd = makeRD(args, opts) rd._metaAttr.copyable = True eTree = base.structToETree(rd) writePrettyPrintedXML(eTree) # ======================= End deprecated ui and implementation =========== def iterColAttrsFITS(table): """yields dicts of column attributes from a FITS binary table HDU. """ hdr = table.header for colInd in range(1, hdr["TFIELDS"]+1): colMeta = [ ("name", re.sub("[^0-9a-z_]", "_", hdr[f"TTYPE{colInd}"].lower())), ("type", getTypeForFTFormat( hdr[f"TFORM{colInd}"], hdr[f"TTYPE{colInd}"])), ("ucd", hdr.get(f"TUCD{colInd}", None)), ("utype", hdr.get(f"TUTYP{colInd}", None)), ("unit", hdr.get(f"TUNIT{colInd}", None)), ("description", hdr.get(f"TCOMM{colInd}", None ) or hdr.cards[f"TTYPE{colInd}"].comment or ""), ("verbLevel", 1),] yield dict((k,v) for k,v in colMeta if v is not None) def iterColAttrsVOTable(inFile, tableInd): """tries to parse (binary) inFile as a VOTable and yields colAttrs dictionaries for all FIELDs in it. """ yielding = tableInd==1 for el in votable.parse(inFile, watchset=[votable.V.TABLE, votable.V.FIELD]): if isinstance(el, votable.V.TABLE): if yielding: break # done parsing our table else: tableInd -= 1 if tableInd==1: yielding = True if yielding and isinstance(el, votable.V.FIELD): colMeta = [ ("name", el.name), ("unit", el.unit), ("ucd", el.ucd or ""), ("utype", el.utype), ("type", base.voTableToSQLType(el.datatype, el.arraysize, el.xtype)), ("description", el.getDescription()), ("verbLevel", 1),] yield dict((k,v) for k,v in colMeta if v is not None) class VizBBBParser(object): """A simple and naive parser for VizieR Byte-by-Byte descriptions. The results are available in the colAttrs and bytesForCols attributes. """ bbbLinePat = re.compile( r"\s*(?P\d+)\s*-\s*(?P\d+)" r"\s+(?P[A-Z0-9.]+)" r"\s+(?P[^\s]*)" r"\s+(?P