2. PyrNet high level data¶

In the following high-level functions to read and examine PyrNet data are collected.

#|export
import pyproj
import numpy as np
import pandas as pd
import xarray as xr
from scipy.interpolate import interp1d
from toolz import valfilter
import pkg_resources as pkg_res

# python -m pip install git+https://github.com/hdeneke/trosat-base.git#egg=trosat-base
from trosat import sunpos as sp

from pyrnet import utils

# extra imports for demonstration
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import datetime as dt

2.2. Read Calibration Files¶

Calibration factors are collected in share/pyrnet_calibration.json. The following function looks up the nearest calibration in time and fill missing values with earlier calibrations

# pyrnet_calibration.json structure
calib = {
    "2017-04":{
        "001":[7.1,7.2],
        "002":[7.51,7.61],
        "003":[6.9,6.91]
    },
    "2019-04": {
        "001":[7,None],
        "002":[7.5,7.6]
    }
}

date1 = np.datetime64("2019-03-01") # closer to "2019-04" calibration
date2 = np.datetime64("2018-01-01") # closer to "2017-04" calibration

# parse calibration dates
cdates = pd.to_datetime(list(calib.keys())).values

# sort calib keys beginning with nearest
skeys1 = np.array(list(calib.keys()))[np.argsort(np.abs(date1 - cdates))]
skeys2 = np.array(list(calib.keys()))[np.argsort(np.abs(date2 - cdates))]
print("Order of calibration lookup")
print(date1, '->' ,skeys1)
print(date2, '->' ,skeys2)

Order of calibration lookup
2019-03-01 -> ['2019-04' '2017-04']
2018-01-01 -> ['2017-04' '2019-04']

Lookup calibration, update with the most recent calibration but fill with earlier calibration if necessary.

# example for date1
for i, key in enumerate(skeys1[::-1]):
    if i==0:
        c = calib[key].copy()
        continue

    isNone = lambda x: np.any([xi is None for xi in x])
    isNotNone = lambda x: np.all([xi is not None for xi in x])
    # update with newer calibrations which not include None values
    c.update(valfilter(isNotNone, calib[key]))

    # update only not None values
    for k,v in valfilter(isNone, calib[key]).items():
        newv = [c[k][i] if vi is None else vi for i,vi in enumerate(v)]
        c.update({k:newv})
c

{'001': [7, 7.2], '002': [7.5, 7.6], '003': [6.9, 6.91]}

# example for date2
for i, key in enumerate(skeys2[::-1]):
    if i==0:
        c = calib[key].copy()
        continue

    isNone = lambda x: np.any([xi is None for xi in x])
    isNotNone = lambda x: np.all([xi is not None for xi in x])
    # update with newer calibrations which not include None values
    c.update(valfilter(isNotNone, calib[key]))

    # update only not None values
    for k,v in valfilter(isNone, calib[key]).items():
        newv = [c[k][i] if vi is None else vi for i,vi in enumerate(v)]
        c.update({k:newv})
c

{'001': [7.1, 7.2], '002': [7.51, 7.61], '003': [6.9, 6.91]}

calib

{'2017-04': {'001': [7.1, 7.2], '002': [7.51, 7.61], '003': [6.9, 6.91]},
 '2019-04': {'001': [7, None], '002': [7.5, 7.6]}}

#|export
def read_calibration(cfile:str, cdate):
    """
    Parse calibration json file

    Parameters
    ----------
    cfile: str
        Path of the calibration.json
    cdate: list, ndarray, or scalar of type float, datetime or datetime64
        A representation of time. If float, interpreted as Julian date.
    Returns
    -------
    dict
        Calibration dictionary sorted by box number.
    """
    cdate = utils.to_datetime64(cdate)
    calib = utils.read_json(cfile)
    # parse calibration dates
    cdates = pd.to_datetime(list(calib.keys())).values
    # sort calib keys beginning with nearest
    skeys = np.array(list(calib.keys()))[np.argsort(np.abs(cdate - cdates))]
    # lookup calibration factors
    for i, key in enumerate(skeys[::-1]):
        if i==0:
            c = calib[key].copy()
            continue
        isNone = lambda x: np.any([xi is None for xi in x])
        isNotNone = lambda x: np.all([xi is not None for xi in x])
        # update with newer calibrations which not include None values
        c.update(valfilter(isNotNone, calib[key]))
        # update only not None values
        for k,v in valfilter(isNone, calib[key]).items():
            newv = [c[k][i] if vi is None else vi for i,vi in enumerate(v)]
            c.update({k:newv})
    return c

Example using the package default pyrnet_calibration.json:

#|dropout
fn = pkg_res.resource_filename("pyrnet", "share/pyrnet_calibration.json")
read_calibration(fn,cdate=np.datetime64("2018-09-10"))

Show code cell output Hide code cell output

{'001': [7.73, 6.98],
 '002': [7.41, None],
 '003': [7.51, None],
 '004': [7.62, 8.23],
 '005': [7.48, 6.37],
 '006': [6.71, None],
 '007': [7.52, 7.2],
 '008': [7.58, None],
 '009': [7.59, 6.85],
 '010': [7.6, 6.74],
 '011': [7.32, None],
 '012': [7.6, 6.87],
 '013': [7.16, None],
 '014': [7.71, 7.3],
 '015': [7.59, 6.77],
 '016': [7.62, None],
 '017': [7.28, None],
 '018': [7.57, 6.9],
 '019': [7.57, None],
 '020': [7.32, None],
 '021': [7.4, None],
 '022': [7.43, None],
 '023': [7.46, None],
 '024': [7.73, 6.59],
 '025': [7.46, 7.14],
 '026': [7.64, 7.05],
 '027': [None, None],
 '028': [7.21, None],
 '029': [7.61, None],
 '030': [7.3, None],
 '031': [7.23, None],
 '032': [7.7, 6.94],
 '033': [7.74, 6.89],
 '034': [6.8, None],
 '035': [7.62, 7.1],
 '036': [None, None],
 '037': [7.72, 6.73],
 '038': [7.62, 6.93],
 '039': [7.27, None],
 '040': [7.15, None],
 '041': [7.38, None],
 '042': [7.15, None],
 '043': [7.41, 7.2],
 '044': [7.26, 7.05],
 '045': [7.52, None],
 '046': [7.7, 7.13],
 '047': [7.47, 7.4],
 '048': [7.1, None],
 '049': [7.74, 7.32],
 '050': [7.51, None],
 '051': [7.67, None],
 '052': [7.46, None],
 '053': [7.63, 6.48],
 '054': [7.58, 7.22],
 '055': [7.41, 6.51],
 '056': [7, None],
 '057': [6.95, None],
 '058': [7.62, 6.9],
 '059': [7.07, None],
 '060': [7.64, 6.62],
 '061': [7.53, None],
 '062': [7.59, None],
 '063': [7.63, None],
 '064': [7.38, None],
 '065': [7.47, None],
 '066': [7.48, None],
 '067': [7.46, None],
 '068': [7.01, None],
 '069': [7.33, None],
 '070': [6.88, None],
 '071': [7.6, None],
 '072': [7.58, None],
 '073': [7.41, None],
 '074': [7.61, None],
 '075': [6.91, None],
 '076': [7.63, None],
 '077': [7.51, None],
 '078': [6.93, None],
 '079': [7.62, None],
 '080': [7.3, None],
 '081': [7.64, None],
 '082': [None, None],
 '083': [7.44, None],
 '084': [7.46, None],
 '085': [7.67, None],
 '086': [7.45, 6.69],
 '087': [7.42, 7.11],
 '088': [7, None],
 '089': [7.59, None],
 '090': [7.64, None],
 '091': [7.47, None],
 '092': [7.35, None],
 '093': [7.51, None],
 '094': [6.95, None],
 '095': [7.47, None],
 '096': [7.65, None],
 '097': [6.9, None],
 '098': [None, None],
 '099': [7.51, None],
 '100': [7.32, None]}

2.3. Read Box Serial numbers¶

Similar to reading the calibration, pyranometers attached to each box are stored in .json format. Reassigning of pyranometers to certain boxes might happen. Different to the calibration we will look up the most recent entry (not in the future)

# pyrnet_calibration.json structure
pyrnetmap = {
    "2019-04": {
        "001":["S11",None],
        "002":["S21","S33"]
    },
    "2017-04":{
        "001":["S11","S21"],
        "002":["S21","S22"],
        "003":["S31",None]
    },
}

date1 = np.datetime64("2019-03-01") # before "2019-04"
date2 = np.datetime64("2019-05-01") # after "2019-04"

# parse key dates
# require sort for lookup later
cdates = pd.to_datetime(list(pyrnetmap.keys())).values
isort = np.argsort(cdates)

# lookup most recent key
skeys1 = np.array(list(pyrnetmap.keys()))[isort][np.sum(date1>cdates)-1]
skeys2 = np.array(list(pyrnetmap.keys()))[isort][np.sum(date2>cdates)-1]
print("Order of calibration lookup")
print(date1, '->' ,skeys1)
print(date2, '->' ,skeys2)

Order of calibration lookup
2019-03-01 -> 2017-04
2019-05-01 -> 2019-04

Now we only have to look it up, no merging needed

#|export
def get_pyrnet_mapping(fn:str, date):
    """
    Parse box - serial number mapping  json file

    Parameters
    ----------
    fn: str
        Path of the mapping.json
    date: list, ndarray, or scalar of type float, datetime or datetime64
        A representation of time. If float, interpreted as Julian date.
    Returns
    -------
    dict
        Calibration dictionary sorted by box number.
    """
    date = utils.to_datetime64(date)
    pyrnetmap = utils.read_json(fn)
    # parse key dates
    # require sort for lookup later
    cdates = pd.to_datetime(list(pyrnetmap.keys())).values
    isort = np.argsort(cdates)

    # lookup most recent key
    skey = np.array(list(pyrnetmap.keys()))[isort][np.sum(date>cdates)-1]

    return pyrnetmap[skey]

2.4. Lookup Serial, Boxnumber, calibration at certain date¶

Utility to lookup box metadata for a certain date.

#|export
def meta_lookup(date,*,serial=None,box=None,cfile=None, mapfile=None):
    if cfile is None:
        cfile = pkg_res.resource_filename("pyrnet", "share/pyrnet_calibration.json")
    if mapfile is None:
        mapfile = pkg_res.resource_filename("pyrnet", "share/pyrnet_station_map.json")

    map = get_pyrnet_mapping(mapfile,date)
    calib = read_calibration(cfile,date)

    if serial is None and box is not None:
        box=int(box)
        return f"{box:03d}", map[f"{box:03d}"], calib[f"{box:03d}"]
    elif serial is not None and box is None:
        res = valfilter(lambda x: serial in x, map)
        box = list(res.keys())[0]
        serial = res[box]
        return box,serial,calib[box]
    else:
        raise ValueError("At least one of [station,box] have to be specified.")

date = np.datetime64("2018-10-01")
print(meta_lookup(date,serial="S12078.061"))
print(meta_lookup(date,box=10))

('006', ['S12078.061', ''], [6.71, None])
('010', ['S12128.010', 'S12137.019'], [7.6, 6.74])

2. PyrNet high level data¶

2.1. Load Data from Thredds-Server¶

2.2. Read Calibration Files¶

2.3. Read Box Serial numbers¶

2.4. Lookup Serial, Boxnumber, calibration at certain date¶

PyrNet

Navigation

Related Topics