{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:27:48.707160Z", "start_time": "2024-08-07T08:27:48.609910Z" }, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "#|default_exp data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Data level\n", "\n", "Reading and advance in data processing levels to be stored in netcdf database format.\n", "\n", "```{note}\n", "raw > l1a > l1b > l1c\n", "```" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:05.914391Z", "start_time": "2024-08-07T08:27:48.751261Z" } }, "outputs": [], "source": [ "#|export\n", "import os\n", "import numpy as np\n", "import pandas as pd\n", "import xarray as xr\n", "import logging\n", "from toolz import assoc_in, merge_with\n", "#import pkg_resources as pkg_res\n", "import importlib.resources\n", "import warnings\n", "\n", "from trosat import sunpos as sp\n", "from trosat.cfconv import read_cfjson\n", "\n", "import pyrnet\n", "pyrnet_version = pyrnet.__version__\n", "import pyrnet.pyrnet\n", "import pyrnet.utils\n", "import pyrnet.logger\n", "import pyrnet.reports\n", "import pyrnet.qcrad\n", "\n", "# logging setup\n", "logging.basicConfig(\n", " filename='pyrnet.log',\n", " encoding='utf-8',\n", " level=logging.DEBUG,\n", " format='%(asctime)s %(name)s %(levelname)s:%(message)s'\n", ")\n", "logger = logging.getLogger(__name__)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.393227Z", "start_time": "2024-08-07T08:28:05.922915Z" } }, "outputs": [], "source": [ "import matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## netCDF dataset operations\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Filename\n", "Filename from config format string." ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.448215Z", "start_time": "2024-08-07T08:28:08.406783Z" } }, "outputs": [], "source": [ "#|export\n", "def get_fname(ds, freq, period=None, timevar='time', sfx='nc', kind=None, station=None, config=None):\n", " config = get_config(config)\n", " if period is None:\n", " period = ds[timevar].values[-1] - ds[timevar].values[0]\n", " period = pd.to_timedelta(period).floor(\"s\").isoformat()\n", " if kind is None:\n", " if ds.station.size==1:\n", " kind = 's'\n", " station = ds.station.values[0]\n", " else:\n", " kind = 'n'\n", " station = ds.station.size\n", " format_dict = dict(\n", " dt = pd.to_datetime(ds[timevar].values[0]),\n", " period = period,\n", " campaign = config[\"campaign\"],\n", " kind = kind,\n", " station = int(station),\n", " level = ds.processing_level,\n", " freq = freq,\n", " collection = int(config[\"collection\"]),\n", " sfx = sfx\n", " )\n", " return config[\"output\"].format(**format_dict) " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Update netcdf coverage metadata\n", "Add and update ACDD covarge metadata to dataset" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.557399Z", "start_time": "2024-08-07T08:28:08.472550Z" } }, "outputs": [], "source": [ "#|export\n", "def update_coverage_meta(ds, timevar='time'):\n", " \"\"\"Update global attributes related to geospatial and time coverage\n", " \"\"\"\n", " duration = ds[timevar].values[-1] - ds[timevar].values[0]\n", " resolution = np.mean(np.diff(ds[timevar].values))\n", " now = pd.to_datetime(np.datetime64(\"now\"))\n", " gattrs = {\n", " 'date_created': now.isoformat(),\n", " 'geospatial_lat_min': np.nanmin(ds.lat.values),\n", " 'geospatial_lat_max': np.nanmax(ds.lat.values),\n", " 'geospatial_lat_units': 'degN',\n", " 'geospatial_lon_min': np.nanmin(ds.lon.values),\n", " 'geospatial_lon_max': np.nanmax(ds.lon.values),\n", " 'geospatial_lon_units': 'degE',\n", " 'time_coverage_start': pd.to_datetime(ds[timevar].values[0]).isoformat(),\n", " 'time_coverage_end': pd.to_datetime(ds[timevar].values[-1]).isoformat(),\n", " 'time_coverage_duration': pd.to_timedelta(duration).isoformat(),\n", " 'time_coverage_resolution': pd.to_timedelta(resolution).isoformat(),\n", " }\n", " ds.attrs.update(gattrs)\n", " return ds\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Stretch resolution\n", "Update encoding *scale_factor* and *add_offset* to use the full dtype resolution." ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.610985Z", "start_time": "2024-08-07T08:28:08.566647Z" } }, "outputs": [], "source": [ "#|export\n", "def stretch_resolution(ds: xr.Dataset) -> xr.Dataset:\n", " \"\"\" Stretch variable resolution to full integer size,\n", " to not lose resolution after averaging ADC count data.\"\"\"\n", " for var in ds:\n", " if \"scale_factor\" not in ds[var].encoding:\n", " continue\n", " if \"valid_range\" not in ds[var].attrs:\n", " continue\n", " dtype = ds[var].encoding['dtype']\n", " valid_range = ds[var].valid_range\n", " int_limit = np.iinfo(dtype).max\n", " scale_factor = ds[var].encoding['scale_factor']\n", " scale_factor_mod = int((int_limit-1)/valid_range[1])\n", " ds[var].encoding.update({\n", " \"scale_factor\": scale_factor / scale_factor_mod,\n", " \"_FillValue\": int_limit,\n", " })\n", " ds[var].attrs.update({\n", " \"valid_range\": valid_range * scale_factor_mod\n", " })\n", " return ds" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Write to netcdf\n", "Ensure complete metadata and merge if needed before writing to netcdf." ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.673319Z", "start_time": "2024-08-07T08:28:08.641672Z" } }, "outputs": [], "source": [ "#|export\n", "def to_netcdf(ds, fname, timevar=\"time\"):\n", " \"\"\"xarray to netcdf, but merge if exist\n", " \"\"\"\n", " # save to netCDF4\n", " ds = update_coverage_meta(ds, timevar=timevar)\n", " ds.to_netcdf(fname,\n", " encoding={timevar:{'dtype':'float64'}}) # for OpenDAP 2 compatibility\n", "#|export\n", "def to_netcdf_l1b(ds, fname, freq='1s', timevar=\"time\"):\n", " \"\"\"xarray to netcdf, but merge if exist\n", " \"\"\"\n", " # merge if necessary\n", " if isinstance(ds, xr.Dataset):\n", " dslist = [ds]\n", " else:\n", " dslist = ds\n", " \n", " if os.path.exists(fname):\n", " ds1 = xr.load_dataset(fname)\n", " dslist.append(ds1)\n", " \n", " ds = merge_l1b(dslist, freq=freq, timevar=timevar)\n", " # save to netCDF4\n", " ds = update_coverage_meta(ds, timevar=timevar)\n", " \n", " if os.path.exists(fname): \n", " os.remove(fname)\n", " ds.to_netcdf(fname,\n", " encoding={timevar:{'dtype':'float64'}}) # for OpenDAP 2 compatibility" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Resample\n", "Specialized (fast) resample methods, as xarray resample is sometimes very slow." ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.714140Z", "start_time": "2024-08-07T08:28:08.680291Z" } }, "outputs": [], "source": [ "#|export\n", "def resample(ds, freq, methods='mean', kwargs={}):\n", " \"\"\" Resample xarray dataset using pandas for speed.\n", " https://github.com/pydata/xarray/issues/4498#issuecomment-706688398\n", " \"\"\"\n", " if isinstance(methods,str):\n", " methods = [methods]\n", "\n", " dsr = ds.to_dataframe().resample(freq)\n", " dsouts = []\n", " for method in methods:\n", " # what we want (quickly), but in Pandas form\n", " with warnings.catch_warnings():\n", " warnings.simplefilter(action='ignore', category=FutureWarning)\n", " df_h = dsr.apply(method)\n", " # rebuild xarray dataset with attributes\n", " vals = []\n", " for c in df_h.columns:\n", " vals.append(\n", " xr.DataArray(data=df_h[c],\n", " dims=['time'],\n", " coords={'time': df_h.index},\n", " attrs=ds[c].attrs)\n", " )\n", " dsouts.append(xr.Dataset(dict(zip(df_h.columns, vals)), attrs=ds.attrs))\n", "\n", " if len(dsouts) == 1:\n", " dsouts = dsouts[0]\n", " return dsouts" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Config \n", "Parse default config" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.766325Z", "start_time": "2024-08-07T08:28:08.727184Z" } }, "outputs": [], "source": [ "#|export\n", "def get_config(config: dict|None = None) -> dict:\n", " \"\"\"Read default config and merge with input config\n", " \"\"\"\n", "\n", " fn_config = os.path.join(importlib.resources.files(\"pyrnet\"), \"share/pyrnet_config.json\")\n", " default_config = pyrnet.utils.read_json(fn_config)\n", " if config is None:\n", " config = default_config\n", " config = {**default_config, **config}\n", "\n", " # add default files\n", " cfiles = {\n", " \"file_cfmeta\": \"share/pyrnet_cfmeta.json\",\n", " \"file_calibration\": \"share/pyrnet_calibration.json\",\n", " \"file_mapping\": \"share/pyrnet_station_map.json\",\n", " \"file_gti_angles\": \"share/pyrnet_gti_angles.json\",\n", " \"file_site\": \"share/pyrnet_sites.json\",\n", " }\n", " for fn in cfiles:\n", " if config[fn] is None:\n", " config[fn] = os.path.join(importlib.resources.files(\"pyrnet\"), cfiles[fn])\n", " return config\n", "\n", "def get_sensor_config(sconfig: dict|None = None) -> dict:\n", " \"\"\" Read the sensor configuration from the default json file and merge if needed. \n", " \"\"\"\n", " fn_config = os.path.join(importlib.resources.files(\"pyrnet\"), \"share/pyrnet_sensor_config.json\")\n", " default_config = pyrnet.utils.read_json(fn_config)\n", " if sconfig is None:\n", " sconfig = default_config\n", " sconfig = {**default_config, **sconfig}\n", " return sconfig\n", "\n", "def get_cfmeta(config: dict|None = None) -> dict:\n", " \"\"\"Read global and variable attributes and encoding from cfmeta.json\n", " \"\"\"\n", " config= get_config(config)\n", " # parse the json file\n", " cfdict = read_cfjson(config[\"file_cfmeta\"])\n", " # get global attributes:\n", " gattrs = cfdict['attributes']\n", " # apply config\n", " gattrs = {k:v.format_map(config) for k,v in gattrs.items()}\n", " # get variable attributes\n", " d = pyrnet.utils.get_var_attrs(cfdict)\n", " # split encoding attributes\n", " vattrs, vencode = pyrnet.utils.get_attrs_enc(d)\n", " return gattrs, vattrs, vencode" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Encoding\n", "The encoding attributes are defined within the file PyrNet/share/pyrnet_cfmeta.json per default.\n", "\n", "The netcdf packing of data is realized via the two attributes *scale_factor* and *add_offset*:\n", " ```unpacked_variable = scale_factor * packed_variable + add_offset```" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.798727Z", "start_time": "2024-08-07T08:28:08.776513Z" } }, "outputs": [], "source": [ "#|export\n", "def calc_encoding(sconfig:dict, ADCV=3.3, ADCbits=10) -> dict:\n", " ADCfac = ADCV / (2**ADCbits-1) # Last bit is reserved \n", " sencoding = {}\n", " for k, v in sconfig.items():\n", " sencoding.update(\n", " {k: dict(\n", " units=v['units'],\n", " scale_factor=v['C']*ADCfac/v['gain'],\n", " add_offset=v['offset'],\n", " valid_range= [0, min(((2**ADCbits-1), int(v['Vmax']*v['gain']/ADCfac)))]\n", " )}\n", " )\n", " return sencoding" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.904118Z", "start_time": "2024-08-07T08:28:08.831580Z" } }, "outputs": [ { "data": { "text/plain": [ "{'ta': {'units': 'K',\n", " 'scale_factor': 0.12903225806451613,\n", " 'add_offset': 253.15,\n", " 'valid_range': [0, 775]},\n", " 'rh': {'units': '1',\n", " 'scale_factor': 0.0012903225806451613,\n", " 'add_offset': 0.0,\n", " 'valid_range': [0, 775]},\n", " 'battery_voltage': {'units': 'V',\n", " 'scale_factor': 0.0064516129032258064,\n", " 'add_offset': 0.0,\n", " 'valid_range': [0, 992]},\n", " 'ghi': {'units': 'V',\n", " 'scale_factor': 1.075268817204301e-05,\n", " 'add_offset': 0.0,\n", " 'valid_range': [0, 1023]},\n", " 'gti': {'units': 'V',\n", " 'scale_factor': 1.075268817204301e-05,\n", " 'add_offset': 0.0,\n", " 'valid_range': [0, 1023]}}" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sconfig = get_sensor_config()\n", "sencoding = calc_encoding(sconfig)\n", "sencoding" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:08.982774Z", "start_time": "2024-08-07T08:28:08.910075Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/plain": [ "{'attributes': {'Conventions': 'CF-1.10, ACDD-1.3',\n", " 'title': 'TROPOS pyranometer network (PyrNet) observational data set',\n", " 'history': '',\n", " 'institution': 'Leibniz Institute for Tropospheric Research (TROPOS)',\n", " 'source': 'TROPOS pyranometer network (PyrNet)',\n", " 'references': 'https://doi.org/10.5194/amt-9-1153-2016',\n", " 'Department': 'Remote Sensing of Atmospheric Processes',\n", " 'Department_team': 'Clouds, Aerosol and Radiation',\n", " 'Address': 'Permoser Str. 15, 04318 Leipzig, Germany',\n", " 'Contact_person': 'Andreas Macke and the clouds, aerosol and radiation team of the remote sensing department, mailto:andreas.macke@tropos.de',\n", " 'Contributor_name': '{contributor_name}',\n", " 'Contributor_role': '{contributor_role}',\n", " 'Authors_software': 'Hartwig Deneke, Jonas Witthuhn, mailto:deneke@tropos.de',\n", " 'Creator_name': '{creator_name}',\n", " 'Project': '{project}',\n", " 'Standard_name_vocabulary': 'CF Standard Name Table v81',\n", " 'License': 'CC-BY-SA 3.0'},\n", " 'variables': {'ta': {'type': 'u2',\n", " 'attributes': {'units': 'K',\n", " 'long_name': 'air temperature',\n", " 'standard_name': 'air_temperature',\n", " 'scale_factor': 0.01,\n", " 'add_offset': 253.15,\n", " '_FillValue': np.uint16(65535),\n", " 'valid_range': [np.uint16(0), np.uint16(10000)],\n", " 'zlib': True}},\n", " 'rh': {'type': 'u2',\n", " 'attributes': {'units': '1',\n", " 'long_name': 'air relative humidity',\n", " 'standard_name': 'relative_humidity',\n", " 'scale_factor': 0.0001,\n", " 'add_offset': 0.0,\n", " 'valid_range': [np.uint16(0), np.uint16(10000)],\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True}},\n", " 'battery_voltage': {'type': 'u2',\n", " 'attributes': {'units': 'V',\n", " 'standard_name': 'battery_voltage',\n", " 'scale_factor': 0.001,\n", " 'add_offset': 0.0,\n", " 'valid_range': [np.uint16(0), np.uint16(6400)],\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True}},\n", " 'ghi': {'type': 'u2',\n", " 'attributes': {'units': 'W m-2',\n", " 'long_name': 'downwelling shortwave flux',\n", " 'standard_name': 'downwelling_shortwave_flux_in_air',\n", " 'scale_factor': 0.025,\n", " 'add_offset': 0.0,\n", " 'valid_range': [np.uint16(0), np.uint16(60000)],\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True}},\n", " 'gti': {'type': 'u2',\n", " 'attributes': {'units': 'V',\n", " 'long_name': 'downwelling shortwave flux measured on secondary platform, might be tilted',\n", " 'standard_name': 'downwelling_shortwave_flux_in_air',\n", " 'scale_factor': 0.025,\n", " 'add_offset': 0.0,\n", " 'valid_range': [np.uint16(0), np.uint16(60000)],\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True}},\n", " 'station': {'type': 'u1',\n", " 'attributes': {'units': '-',\n", " 'long_name': 'PyrNet unit box number',\n", " '_FillValue': np.uint8(255),\n", " 'zlib': True}},\n", " 'szen': {'type': 'u2',\n", " 'attributes': {'standard_name': 'solar_zenith_angle',\n", " 'units': 'degree',\n", " 'scale_factor': 0.005,\n", " 'add_offset': 0.0,\n", " 'valid_range': [np.uint16(0), np.uint16(36000)],\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True}},\n", " 'sazi': {'type': 'u2',\n", " 'attributes': {'standard_name': 'solar_azimuth_angle',\n", " 'units': 'degree',\n", " 'scale_factor': 0.01,\n", " 'add_offset': 0.0,\n", " 'valid_range': [np.uint16(0), np.uint16(36000)],\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True}},\n", " 'esd': {'type': 'f8',\n", " 'attributes': {'long_name': 'Earth-sun distance',\n", " 'note': 'Calculated based on Spencer (1971), as mean of its values over all time steps.',\n", " 'units': 'ua'}},\n", " 'maintenance_flag': {'type': 'u1',\n", " 'attributes': {'standard_name': 'quality_flag',\n", " 'long_name': 'Maintenance quality control flags',\n", " 'note': 'Soiling describes subjectively the coverage of the pyranometer dome with dirt. The level flag is problematic if the bubble of the spirit level touches the reference ring, and bad if it is outside.',\n", " 'valid_range': [np.uint8(0), np.uint8(11)],\n", " 'flag_masks': [np.uint8(3),\n", " np.uint8(3),\n", " np.uint8(3),\n", " np.uint8(12),\n", " np.uint8(12)],\n", " 'flag_values': [np.uint8(1),\n", " np.uint8(2),\n", " np.uint8(3),\n", " np.uint8(4),\n", " np.uint8(8)],\n", " 'flag_meanings': 'soiling_light soiling_moderate soiling_heavy level_problematic level_bad',\n", " '_FillValue': np.uint8(255),\n", " 'zlib': True}},\n", " 'qc_flag': {'type': 'u1',\n", " 'attributes': {'standard_name': 'quality_flag',\n", " 'long_name': 'Automatic quality checks.',\n", " 'valid_range': [np.uint8(0), np.uint8(127)],\n", " 'flag_masks': [np.uint8(1),\n", " np.uint8(2),\n", " np.uint8(4),\n", " np.uint8(8),\n", " np.uint8(16),\n", " np.uint8(32),\n", " np.uint8(64)],\n", " 'flag_values': [np.uint8(1),\n", " np.uint8(2),\n", " np.uint8(4),\n", " np.uint8(8),\n", " np.uint8(16),\n", " np.uint8(32),\n", " np.uint8(64)],\n", " 'flag_meanings': 'below_physical_limit above_physical_limit below_rare_limit above_rare_limit comparison_to_low comparison_to_high quality_control_failed',\n", " '_FillValue': np.uint8(255),\n", " 'zlib': True}},\n", " 'lat': {'type': 'f8',\n", " 'attributes': {'standard_name': 'latitude',\n", " 'units': 'degree_north',\n", " 'valid_range': [np.float64(-90.0), np.float64(90.0)],\n", " '_FillValue': np.float64(-9999.0),\n", " 'zlib': True}},\n", " 'lon': {'type': 'f8',\n", " 'attributes': {'standard_name': 'longitude',\n", " 'units': 'degree_east',\n", " 'valid_range': [np.float64(-180.0), np.float64(180.0)],\n", " '_FillValue': np.float64(-99999.0),\n", " 'zlib': True}},\n", " 'iadc': {'type': 'u4',\n", " 'attributes': {'standard_name': 'index',\n", " 'comment': 'index to map gps to adc records',\n", " 'units': '-',\n", " '_FillValue': np.uint32(4294967295),\n", " 'zlib': True}},\n", " 'adctime': {'type': 'u4',\n", " 'attributes': {'standard_name': 'time',\n", " '_FillValue': np.uint32(4294967295),\n", " 'zlib': True}},\n", " 'time': {'type': 'f8',\n", " 'attributes': {'standard_name': 'time', 'zlib': True}}}}" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "fn_cfjson = os.path.join(importlib.resources.files(\"pyrnet\"), \"share/pyrnet_cfmeta.c01.json\")\n", "read_cfjson(fn_cfjson)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Irradiance values\n", "\n", "Irradiance values are stored as Voltage for later calibration. Assigning 1500Wm-2 as maximum measurable irradiance from the irradiance sensor. The maximum counts (*valid_max*) measured by the logger can be calculated by:\n", " * maximum counts = $\\mathrm{gain}*1500*\\mathrm{C_{max}}*1023/3.3$\n", "\n", "Most calibration factors are in the area of 7.5 uV/Wm-2, assuming $\\mathrm{C_{max}}=8$uV/Wm-2 for this estimate seems sufficient. All radiation sensors are amplified with a fixed gain of $\\mathrm{gain}=300$.\n", "\n", "Later (level 1c+) the scale factor will be instrument specific by adding the calibration from V to W m-2\n", "\n", "### Temperature and humidity\n", "\n", "\n", "Calibration coefficients for Temperature and Humidity DKRF-4000 series (discontinued) https://www.driesen-kern.de/downloads/produktlinie_feuchte.pdf are:\n", " * Temperature (T) range :-20-80 degC from 0-5V\n", " * relative Humidity (rH) range: 0-100% from 0-5V\n", "\n", "As the logger ADC range is 0-3.3V with a 10bit resolution, the sensors are measured through a voltage splitting circuit. Therefore, the ADC counts have to be doubled.\n", " * Voltage U [V] = $2* 3.3/1023$ [counts]\n", " * T = $-20 + 100*(U/5)$ [degC] = $253.15 + 100*(U/5)$ [K]\n", " * rH = $(U/5)$ [%] = $(U/5)$ [%]\n", "\n", "### Ancillary data\n", "\n", "* solar zenith angle (*szen*)\n", " * valid range unpacked: (0,180) (deg)\n", " * packing in u2 integer (unsigned 16bit)\n", " * fill value = $2^{16} - 1$\n", " * scale_factor = $180 /(2^{16}-2)$\n", "* solar azimuth angle (*sazi*)\n", " * valid range unpacked: (0,360)\n", " * packing in u2 integer (unsigned 16bit)\n", " * fill_value = $2^{16} -1$\n", " * scale_factor= $360 / (2^{16}-2)$\n", "* earth sun distance (*esd*)\n", " * valid range unpacked: (0.98,1.02)\n", " * packing in u2 integer (unsigned 16bit)\n", " * fill_value = $2^{16} -1$\n", " * scale_factor= $(1.02-0.98)/(2^{16}-2)$\n", " * add_offset = 0.98\n" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:09.034368Z", "start_time": "2024-08-07T08:28:08.988692Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ta scale_factor: 0.12903225806451613\n", "ta add_offset: 253.15\n", "ta valid_range: [0, 775]\n", "\n", "rh scale_factor: 0.0012903225806451613\n", "rh add_offset: 0\n", "rh valid_range: [0, 775]\n", "\n", "battery scale_factor: 0.0064516129032258064\n", "battery add_offset: 0\n", "battery valid_range: [0, 992]\n", "\n", "radflux scale_factor: 1.075268817204301e-05\n", "radflux add_offset: 0\n", "radflux valid_range: [0, 1023]\n", "\n", "scale_factor szen (deg): 0.005\n", "add_offset szen: 0.0 \n", "valid_range szen: [0, 36000]\n", "\n", "scale_factor sazi (deg): 0.01\n", "add_offset sazi: 0.0 \n", "valid_range sazi: [0, 36000]\n", "\n", "scale_factor esd (AU): 1.0000000000000008e-06\n", "add_offset esd: 0.98 \n", "valid_range esd: [0, 40000]\n", "\n", "scale_factor lat (degree_north): 1e-06\n", "add_offset lat (degree_north): -90.\n", "valid_range lat: [0, 180_000_000]\n", "\n", "scale_factor lon (degree_east): 1e-06\n", "add_offset lon (degree_east): -180.\n", "valid_range lon: [0, 360_000_000]\n", "\n" ] } ], "source": [ "#|dropcode\n", "#|dropout\n", "sensor_config = {\n", " \"ta\": {\n", " \"C\": 100./5., # K/V\n", " \"offset\": 253.15, # K at 0V\n", " \"gain\": 0.5, # measured over voltage splitter\n", " \"Vmax\": 5, # V output at sensor\n", " },\n", " \"rh\": {\n", " \"C\": 1./5., # 1/V\n", " \"offset\": 0, # 0 at 0V\n", " \"gain\": 0.5, # measured over voltage splitter\n", " \"Vmax\": 5, # V output at sensor\n", " },\n", " \"battery\": {\n", " \"C\": 1., # V(batt)/V\n", " \"offset\": 0, # V at 0V\n", " \"gain\": 0.5, # measured over voltage splitter\n", " \"Vmax\": 6.4, # V output at sensor\n", " },\n", " \"radflux\": {\n", " \"C\": 1., # Vsensor/Vmeasured\n", " \"offset\": 0, # V at 0V\n", " \"gain\": 300, # amplified\n", " \"Vmax\": 1500*8*1e-6, # V output at sensor\n", " },\n", "}\n", "ADCfac = 3.3/1023. # V/counts\n", "for var in sensor_config:\n", " sconf = sensor_config[var]\n", " print(f\"{var} scale_factor: {sconf['C']*ADCfac/sconf['gain']}\")\n", " print(f\"{var} add_offset: {sconf['offset']}\")\n", " print(f\"{var} valid_range: [0, {min((1023, int(sconf['Vmax']*sconf['gain']/ADCfac)))}]\")\n", " print()\n", "\n", "# szen encoding\n", "print(f\"scale_factor szen (deg): {180./(36_000)}\")\n", "print(f\"add_offset szen: 0.0 \")\n", "print(f\"valid_range szen: [0, {36_000}]\")\n", "print()\n", "# sazi encoding\n", "print(f\"scale_factor sazi (deg): {360./(36_000)}\")\n", "print(f\"add_offset sazi: 0.0 \")\n", "print(f\"valid_range sazi: [0, {36_000}]\")\n", "print()\n", "# esd encoding\n", "print(f\"scale_factor esd (AU): {(1.02-0.98)/(40_000)}\")\n", "print(f\"add_offset esd: 0.98 \")\n", "print(f\"valid_range esd: [0, {40_000}]\")\n", "print()\n", "# lat encoding\n", "print(f\"scale_factor lat (degree_north): {(90.-(-90.))/180_000_000}\")\n", "print(f\"add_offset lat (degree_north): -90.\")\n", "print(f\"valid_range lat: [0, 180_000_000]\")\n", "print()\n", "# lon encoding\n", "print(f\"scale_factor lon (degree_east): {(180.-(-180.))/360_000_000}\")\n", "print(f\"add_offset lon (degree_east): -180.\")\n", "print(f\"valid_range lon: [0, 360_000_000]\")\n", "print()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Add encoding to netcdf" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:09.130435Z", "start_time": "2024-08-07T08:28:09.063553Z" } }, "outputs": [], "source": [ "#|export\n", "def add_encoding(ds, vencode=None):\n", " \"\"\"\n", " Set valid_range attribute and encoding to every variable of the dataset.\n", "\n", " Parameters\n", " ----------\n", " ds: xr.Dataset\n", " Dataset of any processing level. The processing level will be\n", " determined by the global attribute 'processing_level'.\n", " vencode: dict or None\n", " Dictionary of encoding attributes by variable name, will be merged with pyrnet default cfmeta. The default is None.\n", "\n", " Returns\n", " -------\n", " xr.Dataset\n", " The input dataset but with encoding and valid_range attribute.\n", " \"\"\"\n", " # prepare netcdf encoding\n", " _, vattrs_default, vencode_default = get_cfmeta()\n", "\n", " # Add valid range temporary to encoding dict.\n", " # As valid_range is not implemented in xarray encoding,\n", " # it has to be stored as a variable attribute later.\n", " for k in vencode_default:\n", " if \"valid_range\" in vencode_default[k]:\n", " continue\n", " if \"valid_range\" not in vattrs_default[k]:\n", " continue\n", " vencode_default = assoc_in(vencode_default,\n", " [k,'valid_range'],\n", " vattrs_default[k]['valid_range'])\n", " \n", " # merge input and default with preference on input\n", " if vencode is None:\n", " vencode = vencode_default\n", " else:\n", " a = vencode_default.copy()\n", " b = vencode.copy()\n", " vencode = {}\n", " for k in set(a)-set(b):\n", " vencode.update({k:a[k]})\n", " for k in set(a)&set(b):\n", " vencode.update({k: {**a[k],**b[k]}})\n", " for k in set(b)-set(a):\n", " vencode.update({k:b[k]})\n", "\n", " # add encoding to Dataset\n", " for k, v in vencode.items():\n", " for ki in [key for key in ds if key.startswith(k)]:\n", " ds[ki].encoding.update(v)\n", " if \"valid_range\" not in vencode[k]:\n", " continue\n", " # add valid_range to variable attributes\n", " for ki in [key for key in ds if key.startswith(k)]:\n", " ds[ki].attrs.update({\n", " 'valid_range': vencode[k]['valid_range']\n", " })\n", " \n", " # add encoding to coords\n", " if ds.processing_level=='l1a':\n", " ds[\"gpstime\"].encoding.update({\n", " **vencode[\"time\"],\n", " \"units\": f\"seconds since {np.datetime_as_string(ds.gpstime.data[0], unit='D')}T00:00Z\",\n", " })\n", " ds[\"maintenancetime\"].encoding.update({\n", " **vencode[\"time\"],\n", " \"units\": f\"seconds since {np.datetime_as_string(ds.maintenancetime.data[0], unit='D')}T00:00Z\",\n", " })\n", " ds[\"adctime\"].encoding.update({\n", " **vencode[\"adctime\"],\n", " \"units\": \"milliseconds\"\n", " })\n", " ds[\"station\"].encoding.update({\n", " **vencode[\"station\"]\n", " })\n", " elif ds.processing_level == 'l1b':\n", " ds[\"time\"].encoding.update({\n", " **vencode[\"time\"],\n", " \"units\": f\"seconds since {np.datetime_as_string(ds.time.data[0], unit='D')}T00:00Z\",\n", " })\n", " ds[\"maintenancetime\"].encoding.update({\n", " **vencode[\"time\"],\n", " \"units\": f\"seconds since {np.datetime_as_string(ds.maintenancetime.data[0], unit='D')}T00:00Z\",\n", " })\n", " else:\n", " raise ValueError(\"Dataset has no 'processing_level' attribute.\")\n", " return ds" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## l1a\n", "Full resolution unprocessed, uncalibrated data. One data file corresponds to one maintenance period of one PyrNet station.\n", "\n", "Level *l1a* will be processed from the logger raw data with the following workflow:\n", "\n", "1. Parse raw logger file\n", " * ```pyrnet.logger.read_records```\n", "1. Get maintenance logbook quality flags\n", " * ```pyrnet.reports```\n", "1. Get metadata and encoding\n", " * ```pyrnet_cfmeta_l1b.json```\n", "1. Make xarray Dataset\n", "1. Add variable and global attributes and encoding" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:09.255479Z", "start_time": "2024-08-07T08:28:09.141085Z" } }, "outputs": [], "source": [ "config = get_config()\n", "gattrs, vattrs, vencode = get_cfmeta(config)\n", "\n", "# update encoding with sensor config for l1a\n", "sconfig = get_sensor_config()\n", "sencoding = calc_encoding(sconfig, ADCV=3.3, ADCbits=10)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:09.306453Z", "start_time": "2024-08-07T08:28:09.263953Z" } }, "outputs": [ { "data": { "text/plain": [ "{'ta': {'scale_factor': 0.12903225806451613,\n", " 'add_offset': 253.15,\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True,\n", " 'dtype': 'u2',\n", " 'gzip': True,\n", " 'complevel': 6,\n", " 'units': 'K'},\n", " 'rh': {'scale_factor': 0.0012903225806451613,\n", " 'add_offset': 0.0,\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True,\n", " 'dtype': 'u2',\n", " 'gzip': True,\n", " 'complevel': 6,\n", " 'units': '1'},\n", " 'battery_voltage': {'scale_factor': 0.0064516129032258064,\n", " 'add_offset': 0.0,\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True,\n", " 'dtype': 'u2',\n", " 'gzip': True,\n", " 'complevel': 6,\n", " 'units': 'V'},\n", " 'ghi': {'scale_factor': 1.075268817204301e-05,\n", " 'add_offset': 0.0,\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True,\n", " 'dtype': 'u2',\n", " 'gzip': True,\n", " 'complevel': 6,\n", " 'units': 'V'},\n", " 'gti': {'scale_factor': 1.075268817204301e-05,\n", " 'add_offset': 0.0,\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True,\n", " 'dtype': 'u2',\n", " 'gzip': True,\n", " 'complevel': 6,\n", " 'units': 'V'},\n", " 'station': {'_FillValue': np.uint8(255),\n", " 'zlib': True,\n", " 'dtype': 'u1',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'szen': {'scale_factor': 0.005,\n", " 'add_offset': 0.0,\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True,\n", " 'dtype': 'u2',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'sazi': {'scale_factor': 0.01,\n", " 'add_offset': 0.0,\n", " '_FillValue': np.uint16(65535),\n", " 'zlib': True,\n", " 'dtype': 'u2',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'esd': {'dtype': 'f8', 'gzip': True, 'complevel': 6},\n", " 'maintenance_flag': {'_FillValue': np.uint8(255),\n", " 'zlib': True,\n", " 'dtype': 'u1',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'qc_flag': {'_FillValue': np.uint8(255),\n", " 'zlib': True,\n", " 'dtype': 'u1',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'lat': {'_FillValue': np.float64(-9999.0),\n", " 'zlib': True,\n", " 'dtype': 'f8',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'lon': {'_FillValue': np.float64(-99999.0),\n", " 'zlib': True,\n", " 'dtype': 'f8',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'iadc': {'_FillValue': np.uint32(4294967295),\n", " 'zlib': True,\n", " 'dtype': 'u4',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'adctime': {'_FillValue': np.uint32(4294967295),\n", " 'zlib': True,\n", " 'dtype': 'u4',\n", " 'gzip': True,\n", " 'complevel': 6},\n", " 'time': {'zlib': True, 'dtype': 'f8', 'gzip': True, 'complevel': 6}}" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# from toolz import update_in\n", "for var,enc in sencoding.items():\n", " for k,v in enc.items():\n", " if k==\"valid_range\":\n", " vattrs = assoc_in(vattrs, [var,k], v)\n", " else:\n", " vencode = assoc_in(vencode, [var,k], v)\n", "vencode" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## to l1a_function" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:09.431862Z", "start_time": "2024-08-07T08:28:09.315087Z" } }, "outputs": [], "source": [ "#|export\n", "def to_l1a(\n", " fname : str,\n", " *,\n", " station: int,\n", " report: dict|pd.DataFrame|None,\n", " date_of_measure : np.datetime64 = np.datetime64(\"now\"),\n", " config: dict|None = None,\n", " sconfig: dict|None = None,\n", " global_attrs: dict|None = None\n", ") -> xr.Dataset|None:\n", " \"\"\"\n", " Read logger raw file and parse it to xarray Dataset. Thereby, attributes and names are defined via cfmeta.json file and sun position values are calculated and added.\n", "\n", " Parameters\n", " ----------\n", " fname: str\n", " Path and filename of the raw logger file.\n", " station: int\n", " PyrNet station box number.\n", " report: dict\n", " Parsed maintenance report, see reports.ipynb\n", " bins: int\n", " Number of desired bins per day. The default is 86400, which result in\n", " mean values of 1 second steps per day. Maximum resolution is 86400000.\n", " date_of_measure: float, datetime or datetime64\n", " A rough date of measure to account for GPS week rollover. If measured in 2019, day resolution is recommended, before 2019 annual resolution, 2020 onwards not required. If float, interpreted as Julian day from 2000-01-01T12:00. the default is np.datetime64(\"now\").\n", " config: dict\n", " Stores processing specific configuration.\n", " * cfjson -> path to cfmeta.json, the default is \"../share/pyrnet_cfmeta.json\"\n", " * stripminutes -> number of minutes to be stripped from the data at start and end,\n", " the default is 5.\n", " sconfig: dict\n", " Config for ADC and amplifier for each sensor. The default is \"../share/pyrnet_sensor_config.json\"\n", " global_attrs: dict\n", " Additional global attributes for the Dataset. (Overrides cfmeta.json attributes)\n", " Returns\n", " -------\n", " xarray.Dataset\n", " Raw Logger data for one measurement periode.\n", " \"\"\"\n", " ADCV = 3.3\n", " ADCbits = 10\n", " \n", " # load and merge default config\n", " config = get_config(config)\n", " gattrs, vattrs, vencode = get_cfmeta(config)\n", " \n", " # update encoding with sensor config for l1a\n", " sconfig = get_sensor_config(sconfig)\n", " sencoding = calc_encoding(sconfig, ADCV=ADCV, ADCbits=ADCbits)\n", " # update encoding with sensor config for l1a\n", " for var, enc in sencoding.items():\n", " for k, v in enc.items():\n", " if k in [\"units\"]:\n", " vattrs = assoc_in(vattrs, [var, k], v)\n", " else:\n", " vencode = assoc_in(vencode, [var, k], v)\n", " \n", " # update additional global attributes\n", " if global_attrs is not None:\n", " gattrs.update(global_attrs)\n", "\n", " date_of_measure = pyrnet.utils.to_datetime64(date_of_measure)\n", "\n", " # 1. Parse raw file\n", " rec_adc, rec_gprmc = pyrnet.logger.read_records(fname=fname, date_of_measure=date_of_measure)\n", "\n", " if type(rec_adc)==bool or len(rec_gprmc.time)<3:\n", " logger.debug(\"Failed to load the data from the file, because of not enough stable GPS data, or file is empty.\")\n", " return None\n", "\n", " # Get ADC time\n", " adctime = pyrnet.logger.get_adc_time(rec_adc)\n", "\n", " # ADC to Volts\n", " # Drop time and internal battery sensor output (columns 0 and 1)\n", " adc_volts = ADCV * rec_adc[:,2:] / float(2**ADCbits - 1)\n", "\n", " # 2. Get Logbook maintenance quality flags\n", " key = f\"{station:03d}\"\n", " if report is None:\n", " logger.warning(\"No report available!\")\n", " report = {}\n", " if isinstance(report, pd.DataFrame):\n", " logger.info(f\"Parsing report at date {rec_gprmc.time[-1]}\")\n", " report = pyrnet.reports.parse_report(\n", " report,\n", " date_of_maintenance=rec_gprmc.time[-1],\n", " stations=station\n", " )\n", "\n", " if key not in report:\n", " logger.warning(f\"No report for station {station} available.\")\n", " warnings.warn(f\"No report for station {station} available.\")\n", " qc_main = pyrnet.reports.get_qcflag(4,3)\n", " qc_extra = pyrnet.reports.get_qcflag(4,3)\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_ghi\",\"note_general\"], \"No maintenance report!\")\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_gti\",\"note_general\"], \"No maintenance report!\")\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_ghi\",\"note_clean\"], \"\")\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_gti\",\"note_clean\"], \"\")\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_ghi\",\"note_level\"], \"\")\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_gti\",\"note_level\"], \"\")\n", " maintenancetime = np.array([rec_gprmc.time.astype('datetime64[ns]')[-1]])\n", " else:\n", " qc_main = pyrnet.reports.get_qcflag(\n", " qc_clean=report[key]['clean'],\n", " qc_level=report[key]['align']\n", " )\n", " qc_extra = pyrnet.reports.get_qcflag(\n", " qc_clean=report[key]['clean2'],\n", " qc_level=report[key]['align2']\n", " )\n", " maintenancetime = np.array([pyrnet.utils.to_datetime64(report[key][\"maintenancetime\"])]) \n", " # add qc notes\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_ghi\",\"note_general\"], report[key][\"note_general\"])\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_gti\",\"note_general\"], report[key][\"note_general\"])\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_ghi\",\"note_clean\"], report[key][\"note_clean\"])\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_gti\",\"note_clean\"], report[key][\"note_clean2\"])\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_ghi\",\"note_level\"], report[key][\"note_align\"])\n", " vattrs = assoc_in(vattrs, [\"maintenance_flag_gti\",\"note_level\"], report[key][\"note_align2\"])\n", " qc_main = np.ubyte(qc_main)\n", " qc_extra = np.ubyte(qc_extra)\n", " \n", " vattrs = assoc_in(vattrs, [\"ghi\",\"ancillary_variables\"], \"maintenance_flag_ghi\")\n", " vattrs = assoc_in(vattrs, [\"gti\",\"ancillary_variables\"], \"maintenance_flag_gti\")\n", "\n", " # 3. Add global meta data\n", " now = pd.to_datetime(np.datetime64(\"now\"))\n", " gattrs.update({\n", " 'processing_level': 'l1a',\n", " 'product_version': pyrnet_version,\n", " 'history': f'{now.isoformat()}: Generated level l1a by pyrnet version {pyrnet_version}; ',\n", " })\n", " # add site information\n", " if config['sites'] is not None:\n", " sites = pyrnet.utils.read_json(config['file_site'])[config['sites']]\n", " if key in sites:\n", " gattrs.update({ \"site\" : sites[key]})\n", "\n", " # add gti angles\n", " # default horizontal\n", " vattrs = assoc_in(vattrs, [\"gti\",\"hangle\"], 0.)\n", " vattrs = assoc_in(vattrs, [\"gti\",\"vangle\"], 0.)\n", " # update with angles from mapping file\n", " if config['gti_angles'] is not None:\n", " gti_angles = pyrnet.utils.read_json(config['file_gti_angles'])[config['gti_angles']]\n", " if key in gti_angles:\n", " hangle = np.nan if gti_angles[key][0] is None else gti_angles[key][0]\n", " vangle = np.nan if gti_angles[key][1] is None else gti_angles[key][1]\n", " vattrs = assoc_in(vattrs, [\"gti\",\"hangle\"], hangle)\n", " vattrs = assoc_in(vattrs, [\"gti\",\"vangle\"], vangle)\n", "\n", " if adc_volts.shape[1]<5: # gti data is not available\n", " adc_volts = np.concatenate((adc_volts,-1*np.ones(adc_volts.shape[0])[:,None]),axis=1)\n", "\n", " \n", "\n", " # 8. Make xarray Dataset\n", " values = {}\n", " for k, v in sconfig.items():\n", " offset = sconfig[k][\"offset\"]\n", " gain = sconfig[k][\"gain\"]\n", " C = sconfig[k][\"C\"]\n", " iadc = sconfig[k][\"iadc\"]\n", " volts = adc_volts[:,iadc][:,None]\n", " values.update(\n", " {k: offset + C*volts/gain}\n", " )\n", " \n", " # Remove rh and temperature error if input voltage to low\n", " # Allow only sensor values of voltage V < (V(battery)-2),\n", " # as input voltage needs to be at least 2V larger then sensor output\n", " for k in [\"rh\",\"ta\"]:\n", " iadc = sconfig[k][\"iadc\"]\n", " volts_sensor = (values[k][:,0]-sconfig[k][\"offset\"])/sconfig[k][\"C\"] # no gain, as we need voltage at sensor\n", " mask = volts_sensor > (values[\"battery_voltage\"][:,0] - 2.) \n", " values[k][mask,0] = np.nan \n", " \n", " ds = xr.Dataset(\n", " data_vars={\n", " \"ghi\": ((\"adctime\",\"station\"), values[\"ghi\"]), # [V]\n", " \"gti\": ((\"adctime\",\"station\"), values[\"gti\"]), # [V]\n", " \"ta\": ((\"adctime\",\"station\"), values[\"ta\"]), # [K]\n", " \"rh\": ((\"adctime\",\"station\"), values[\"rh\"]), # [-]\n", " \"battery_voltage\": ((\"adctime\",\"station\"), values[\"battery_voltage\"]), # [V]\n", " \"lat\": ((\"gpstime\",\"station\"), rec_gprmc.lat[:,None]), # [degN]\n", " \"lon\": ((\"gpstime\",\"station\"), rec_gprmc.lon[:,None]), # [degE]\n", " \"maintenance_flag_ghi\": ((\"maintenancetime\",\"station\"), [[qc_main]]),\n", " \"maintenance_flag_gti\": ((\"maintenancetime\",\"station\"), [[qc_extra]]),\n", " \"iadc\": ((\"gpstime\", \"station\"), rec_gprmc.iadc[:,None])\n", " },\n", " coords={\n", " \"adctime\": (\"adctime\", adctime.astype('timedelta64[ns]')),\n", " \"gpstime\": (\"gpstime\", rec_gprmc.time.astype('datetime64[ns]')),\n", " \"maintenancetime\": (\"maintenancetime\", maintenancetime),\n", " \"station\": (\"station\", [np.ubyte(station)]),\n", " },\n", " attrs=gattrs\n", " )\n", "\n", " # drop ocurance of douplicate gps values\n", " ds = ds.drop_duplicates(\"gpstime\")\n", "\n", " # add global coverage attributes\n", " ds = update_coverage_meta(ds, timevar=\"gpstime\")\n", "\n", " # add attributes to Dataset\n", " for k,v in vattrs.items():\n", " for key in [key for key in ds if key.startswith(k)]:\n", " ds[key].attrs.update(v)\n", "\n", " # add encoding to Dataset\n", " ds = add_encoding(ds, vencode)\n", "\n", " return ds" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Test to_l1a function" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:09.983931Z", "start_time": "2024-08-07T08:28:09.439294Z" }, "tags": [ "hide-output" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Processing fname:\n", "2022-08-30_P0DT0H0M4S_pyrnet__s001l1af10Hz.c01.nc\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 4kB\n",
       "Dimensions:               (adctime: 77, station: 1, gpstime: 5,\n",
       "                           maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * adctime               (adctime) timedelta64[ns] 616B 00:00:00 ... 00:00:0...\n",
       "  * gpstime               (gpstime) datetime64[ns] 40B 2022-08-30T11:21:04.06...\n",
       "  * maintenancetime       (maintenancetime) datetime64[ms] 8B 2023-05-08T16:0...\n",
       "  * station               (station) uint8 1B 1\n",
       "Data variables:\n",
       "    ghi                   (adctime, station) float64 616B 0.002086 ... 0.002086\n",
       "    gti                   (adctime, station) float64 616B 0.001935 ... 0.001935\n",
       "    ta                    (adctime, station) float64 616B 294.7 294.7 ... 295.0\n",
       "    rh                    (adctime, station) float64 616B 0.6245 ... 0.6297\n",
       "    battery_voltage       (adctime, station) float64 616B 6.452 6.439 ... 6.452\n",
       "    lat                   (gpstime, station) float64 40B 51.39 51.39 ... 51.39\n",
       "    lon                   (gpstime, station) float64 40B 11.89 11.89 ... 11.89\n",
       "    maintenance_flag_ghi  (maintenancetime, station) uint8 1B 9\n",
       "    maintenance_flag_gti  (maintenancetime, station) uint8 1B 7\n",
       "    iadc                  (gpstime, station) uint32 20B 26 35 54 64 74\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256666666667\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:04.065000\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M4.935S\n",
       "    time_coverage_resolution:  P0DT0H0M1.23375S
" ], "text/plain": [ " Size: 4kB\n", "Dimensions: (adctime: 77, station: 1, gpstime: 5,\n", " maintenancetime: 1)\n", "Coordinates:\n", " * adctime (adctime) timedelta64[ns] 616B 00:00:00 ... 00:00:0...\n", " * gpstime (gpstime) datetime64[ns] 40B 2022-08-30T11:21:04.06...\n", " * maintenancetime (maintenancetime) datetime64[ms] 8B 2023-05-08T16:0...\n", " * station (station) uint8 1B 1\n", "Data variables:\n", " ghi (adctime, station) float64 616B 0.002086 ... 0.002086\n", " gti (adctime, station) float64 616B 0.001935 ... 0.001935\n", " ta (adctime, station) float64 616B 294.7 294.7 ... 295.0\n", " rh (adctime, station) float64 616B 0.6245 ... 0.6297\n", " battery_voltage (adctime, station) float64 616B 6.452 6.439 ... 6.452\n", " lat (gpstime, station) float64 40B 51.39 51.39 ... 51.39\n", " lon (gpstime, station) float64 40B 11.89 11.89 ... 11.89\n", " maintenance_flag_ghi (maintenancetime, station) uint8 1B 9\n", " maintenance_flag_gti (maintenancetime, station) uint8 1B 7\n", " iadc (gpstime, station) uint32 20B 26 35 54 64 74\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropout\n", "fn_report = \"../../example_data/results-survey224783.csv\"\n", "fn_data = \"../../example_data/Pyr9_000.bin\"\n", "\n", "fn_cfmeta = os.path.join(importlib.resources.files(\"pyrnet\"), \"share/pyrnet_cfmeta.c01.json\")\n", "\n", "\n", "# parse report\n", "df_report = pyrnet.reports.get_responses(fn=\"../../example_data/results-survey224783.csv\")\n", "report = pyrnet.reports.parse_report(df_report,\n", " date_of_maintenance=np.datetime64(\"2023-05-08T12:00\"))\n", "# read logger file to xarray\n", "ds = to_l1a(\n", " fname=fn_data,\n", " station=1, # actually test data is from station 9, but test reports are for station 1 and 2 only\n", " # bins=86400, # seconds resolution\n", " report=report,\n", " config={\"file_cfmeta\": fn_cfmeta, \"stripminutes\": 0},\n", " global_attrs={\"TESTNOTE\": \"This is a test note.\"}\n", ")\n", "\n", "print(\"Processing fname:\")\n", "print(get_fname(ds, freq=\"10Hz\", timevar=\"gpstime\", config=None))\n", "\n", "ds.to_netcdf(\"../../example_data/to_l1a_output.nc\")\n", "ds" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:11.280381Z", "start_time": "2024-08-07T08:28:09.989720Z" } }, "outputs": [], "source": [ "# import cfchecker.cfchecks\n", "# init = cfchecker.cfchecks.CFChecker()\n", "# res = init.checker(\"../../example_data/to_l1a_output.nc\")" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:11.381404Z", "start_time": "2024-08-07T08:28:11.285091Z" }, "tags": [ "hide-output" ] }, "outputs": [ { "data": { "text/plain": [ "\n", "root group (NETCDF4 data model, file format HDF5):\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observational data set\n", " history: 2025-10-16T11:47:26: Generated level l1a by pyrnet version 1.0.0+1.g74c8a89; \n", " institution: Leibniz Institute for Tropospheric Research (TROPOS)\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " Department: Remote Sensing of Atmospheric Processes\n", " Department_team: Clouds, Aerosol and Radiation\n", " Address: Permoser Str. 15, 04318 Leipzig, Germany\n", " Contact_person: Andreas Macke and the clouds, aerosol and radiation team of the remote sensing department, mailto:andreas.macke@tropos.de\n", " Contributor_name: \n", " Contributor_role: \n", " Authors_software: Hartwig Deneke, Jonas Witthuhn, mailto:deneke@tropos.de\n", " Creator_name: \n", " Project: \n", " Standard_name_vocabulary: CF Standard Name Table v81\n", " License: CC-BY-SA 3.0\n", " TESTNOTE: This is a test note.\n", " processing_level: l1a\n", " product_version: 1.0.0+1.g74c8a89\n", " date_created: 2025-10-16T11:47:26\n", " geospatial_lat_min: 51.390208333333334\n", " geospatial_lat_max: 51.390211666666666\n", " geospatial_lat_units: degN\n", " geospatial_lon_min: 11.885246666666667\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S\n", " dimensions(sizes): adctime(77), station(1), gpstime(5), maintenancetime(1)\n", " variables(dimensions): uint16 ghi(adctime, station), uint16 gti(adctime, station), uint16 ta(adctime, station), uint16 rh(adctime, station), uint16 battery_voltage(adctime, station), float64 lat(gpstime, station), float64 lon(gpstime, station), uint8 maintenance_flag_ghi(maintenancetime, station), uint8 maintenance_flag_gti(maintenancetime, station), uint32 iadc(gpstime, station), uint32 adctime(adctime), float64 gpstime(gpstime), float64 maintenancetime(maintenancetime), uint8 station(station)\n", " groups: " ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropout\n", "import netCDF4\n", "netCDF4.Dataset(\"../../example_data/to_l1a_output.nc\",'r')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "\n", "\n", "## l1b\n", "Resampled, calibrated data as daily files per station. Resampling is per default to one second, but can be configured.\n" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:11.432190Z", "start_time": "2024-08-07T08:28:11.387746Z" } }, "outputs": [], "source": [ "fname = \"../../example_data/to_l1a_output.nc\"\n", "config = {\n", " \"l1bfreq\": \"1s\",\n", " \"stripminutes\": 5,\n", " \"average_latlon\": True,\n", " \"radflux_varname\": [\"ghi\", \"gti\"]\n", "}\n", "\n", "config = get_config(config)\n", "gattrs, vattrs, vencode = get_cfmeta(config)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The l1b data is processed from the l1b data with the following workflow:\n", "\n", "### Read l1a netcdf\n", "Ensure its l1a by checking *processing_level* attribute.\n", "\n", "### Sync GPS to ADC time\n", "Use ```sync_adc_time``` to fit logger ADC timedelta since operation start to GPS timestamps. This is crucial to overcome time drifts and ensure synchronized measurements.\n", "\n", "### Make dataset with new time\n", "Initialize new l1b dataset with the synchronized time as coordinate" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:11.928826Z", "start_time": "2024-08-07T08:28:11.439463Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 4kB\n",
       "Dimensions:          (time: 77, station: 1, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * maintenancetime  (maintenancetime) datetime64[ns] 8B 2023-05-08T16:09:06\n",
       "  * station          (station) float32 4B 1.0\n",
       "  * time             (time) datetime64[ns] 616B 2022-08-30T11:21:01.443000 .....\n",
       "Data variables:\n",
       "    ghi              (time, station) float64 616B 0.002086 0.002086 ... 0.002086\n",
       "    gti              (time, station) float64 616B 0.001935 0.001935 ... 0.001935\n",
       "    ta               (time, station) float64 616B 294.7 294.7 ... 294.8 295.0\n",
       "    rh               (time, station) float64 616B 0.6245 0.6245 ... 0.6297\n",
       "    battery_voltage  (time, station) float64 616B 6.452 6.439 ... 6.465 6.452\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256666666667\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:04.065000\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M4.935S\n",
       "    time_coverage_resolution:  P0DT0H0M1.23375S
" ], "text/plain": [ " Size: 4kB\n", "Dimensions: (time: 77, station: 1, maintenancetime: 1)\n", "Coordinates:\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:09:06\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 616B 2022-08-30T11:21:01.443000 .....\n", "Data variables:\n", " ghi (time, station) float64 616B 0.002086 0.002086 ... 0.002086\n", " gti (time, station) float64 616B 0.001935 0.001935 ... 0.001935\n", " ta (time, station) float64 616B 294.7 294.7 ... 294.8 295.0\n", " rh (time, station) float64 616B 0.6245 0.6245 ... 0.6297\n", " battery_voltage (time, station) float64 616B 6.452 6.439 ... 6.465 6.452\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "# 1. Load l1a data\n", "ds_l1a = xr.load_dataset(fname)\n", "# check correct file\n", "if ds_l1a.processing_level != \"l1a\":\n", " raise ValueError(f\"{fname} is not a l1a file.\")\n", "\n", "\n", "# 2. Sync GPS to ADC time\n", "adctime = pyrnet.logger.sync_adc_time(\n", " adctime = ds_l1a.adctime.values,\n", " gpstime = ds_l1a.gpstime.values,\n", " iadc = ds_l1a.iadc.squeeze().values.astype(int),\n", " check_results= False\n", ")\n", "\n", "# 3. Create new dataset (l1b) with synced time\n", "\n", "ds_l1b = ds_l1a.drop_dims('gpstime')\n", "ds_l1b = ds_l1b.drop_vars(['maintenance_flag_ghi','maintenance_flag_gti']) # keep only time dependend variables\n", "ds_l1b = ds_l1b.assign({'time': ('adctime', adctime)})\n", "ds_l1b = ds_l1b.swap_dims({\"adctime\":\"time\"})\n", "ds_l1b = ds_l1b.drop_vars(\"adctime\")\n", "ds_l1b" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:11.954742Z", "start_time": "2024-08-07T08:28:11.937527Z" }, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "# dsp = ds_l1b.sel(time=\"2019-07-15\")\n", "# plt.figure()\n", "# plt.plot(dsp.time, dsp.ghi)\n", "# plt.grid()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Drop first and last X minutes of data \n", "To avoid bad data due to maintenance during start or end of the maintenance interval, a certain amound of minutes is dropped (defined in config).\n", "\n", "```{note}\n", "config.json -> \"stripminutes\"\n", "```" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:12.233518Z", "start_time": "2024-08-07T08:28:11.975180Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 2kB\n",
       "Dimensions:          (time: 37, station: 1, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * maintenancetime  (maintenancetime) datetime64[ns] 8B 2023-05-08T16:09:06\n",
       "  * station          (station) float32 4B 1.0\n",
       "  * time             (time) datetime64[ns] 296B 2022-08-30T11:21:03.491000 .....\n",
       "Data variables:\n",
       "    ghi              (time, station) float64 296B 0.002086 0.002086 ... 0.002086\n",
       "    gti              (time, station) float64 296B 0.001935 0.001935 ... 0.001935\n",
       "    ta               (time, station) float64 296B 294.7 294.7 ... 294.8 294.8\n",
       "    rh               (time, station) float64 296B 0.6245 0.6258 ... 0.6297\n",
       "    battery_voltage  (time, station) float64 296B 6.452 6.452 ... 6.452 6.465\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256666666667\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:04.065000\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M4.935S\n",
       "    time_coverage_resolution:  P0DT0H0M1.23375S
" ], "text/plain": [ " Size: 2kB\n", "Dimensions: (time: 37, station: 1, maintenancetime: 1)\n", "Coordinates:\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:09:06\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 296B 2022-08-30T11:21:03.491000 .....\n", "Data variables:\n", " ghi (time, station) float64 296B 0.002086 0.002086 ... 0.002086\n", " gti (time, station) float64 296B 0.001935 0.001935 ... 0.001935\n", " ta (time, station) float64 296B 294.7 294.7 ... 294.8 294.8\n", " rh (time, station) float64 296B 0.6245 0.6258 ... 0.6297\n", " battery_voltage (time, station) float64 296B 6.452 6.452 ... 6.452 6.465\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "# 4. Drop first and last minutes of data to avoid bad data due to maintenance\n", "\n", "# stripminutes = np.timedelta64(int(config['stripminutes']), 'm')\n", "# for now use some seconds for demonstration\n", "stripminutes = np.timedelta64(2, 's')\n", "tslice = slice(ds_l1b.time.values[0] + stripminutes,\n", " ds_l1b.time.values[-1] - stripminutes)\n", "ds_l1b = ds_l1b.sel(time=tslice)\n", "ds_l1b" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Resample\n", "Resample the data to desired time interval\n", " * ```pyrnet.data.resample```\n" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:12.589824Z", "start_time": "2024-08-07T08:28:12.245343Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 492B\n",
       "Dimensions:          (station: 1, time: 5, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * station          (station) float32 4B 1.0\n",
       "  * time             (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2022-0...\n",
       "  * maintenancetime  (maintenancetime) datetime64[ns] 8B 2023-05-08T16:09:06\n",
       "Data variables:\n",
       "    ghi              (time, station) float64 40B 0.002086 0.002085 ... 0.002086\n",
       "    gti              (time, station) float64 40B 0.001935 0.001935 ... 0.001935\n",
       "    ta               (time, station) float64 40B 294.8 294.8 294.8 294.8 294.8\n",
       "    rh               (time, station) float64 40B 0.6263 0.6266 ... 0.6271 0.629\n",
       "    battery_voltage  (time, station) float64 40B 6.445 6.446 6.45 6.448 6.458\n",
       "    ghi_min          (time, station) float64 40B 0.002086 0.002075 ... 0.002086\n",
       "    gti_min          (time, station) float64 40B 0.001935 0.001925 ... 0.001935\n",
       "    ghi_max          (time, station) float64 40B 0.002086 0.002086 ... 0.002086\n",
       "    gti_max          (time, station) float64 40B 0.001935 0.001946 ... 0.001935\n",
       "    ghi_std          (time, station) float64 40B 0.0 3.4e-06 0.0 0.0 0.0\n",
       "    gti_std          (time, station) float64 40B 0.0 5.069e-06 6.104e-06 0.0 0.0\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256666666667\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:04.065000\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M4.935S\n",
       "    time_coverage_resolution:  P0DT0H0M1.23375S
" ], "text/plain": [ " Size: 492B\n", "Dimensions: (station: 1, time: 5, maintenancetime: 1)\n", "Coordinates:\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2022-0...\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:09:06\n", "Data variables:\n", " ghi (time, station) float64 40B 0.002086 0.002085 ... 0.002086\n", " gti (time, station) float64 40B 0.001935 0.001935 ... 0.001935\n", " ta (time, station) float64 40B 294.8 294.8 294.8 294.8 294.8\n", " rh (time, station) float64 40B 0.6263 0.6266 ... 0.6271 0.629\n", " battery_voltage (time, station) float64 40B 6.445 6.446 6.45 6.448 6.458\n", " ghi_min (time, station) float64 40B 0.002086 0.002075 ... 0.002086\n", " gti_min (time, station) float64 40B 0.001935 0.001925 ... 0.001935\n", " ghi_max (time, station) float64 40B 0.002086 0.002086 ... 0.002086\n", " gti_max (time, station) float64 40B 0.001935 0.001946 ... 0.001935\n", " ghi_std (time, station) float64 40B 0.0 3.4e-06 0.0 0.0 0.0\n", " gti_std (time, station) float64 40B 0.0 5.069e-06 6.104e-06 0.0 0.0\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "# 6. resample to desired resolution\n", "# save station coordinate\n", "station_dim = {\"station\": ds_l1b[\"station\"].values}\n", "station_attrs = ds_l1b[\"station\"].attrs\n", "\n", "# resample on time dimension with specified methods\n", "methods = ['mean'] + config[\"l1b_resample_stats\"]\n", "res = resample(\n", " ds_l1b.squeeze().drop_vars(\"station\"), # drop station coordinate and variable\n", " freq=config['l1bfreq'],\n", " methods=methods,\n", " kwargs=dict(skipna=True)\n", ")\n", "\n", "# add standard names for new variables\n", "ds_l1b = res[0]\n", "for i, method in enumerate(methods[1:]):\n", " for var in config[\"radflux_varname\"]:\n", " ds_l1b[f\"{var}_{method}\"] = res[i+1][var]\n", " ds_l1b[f\"{var}_{method}\"].attrs.update({\n", " \"standard_name\": f\"{method}_\"+ds_l1b[f\"{var}_{method}\"].attrs[\"standard_name\"]\n", " })\n", "\n", "# add station dimension back again\n", "ds_l1b = ds_l1b.expand_dims(station_dim, axis=-1)\n", "ds_l1b[\"station\"].attrs.update(station_attrs)\n", "\n", "# add maintenancetime coord\n", "ds_l1b = ds_l1b.assign_coords({\"maintenancetime\":ds_l1a.maintenancetime})\n", "ds_l1b" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:12.612191Z", "start_time": "2024-08-07T08:28:12.596952Z" }, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "# dsp = ds_l1b.sel(time=\"2019-07-15\")\n", "# plt.figure()\n", "# plt.plot(dsp.time, dsp.ghi)\n", "# plt.grid()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Interpolate GPS coordinates\n", "\n", "Use ```xarray.interp``` to interpolate lat and lon to the resampled time dimension.\n", "```{note}\n", "At this point the descision to whether store geocoordinates in full time resolution or as mean over the time interval is made.\n", "\n", "This is configured in config.json -> \"average_latlon\"\n", "```" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:12.962969Z", "start_time": "2024-08-07T08:28:12.629532Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 580B\n",
       "Dimensions:               (station: 1, time: 5, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * station               (station) float32 4B 1.0\n",
       "  * time                  (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n",
       "  * maintenancetime       (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
       "Data variables: (12/15)\n",
       "    ghi                   (time, station) float64 40B 0.002086 ... 0.002086\n",
       "    gti                   (time, station) float64 40B 0.001935 ... 0.001935\n",
       "    ta                    (time, station) float64 40B 294.8 294.8 ... 294.8\n",
       "    rh                    (time, station) float64 40B 0.6263 0.6266 ... 0.629\n",
       "    battery_voltage       (time, station) float64 40B 6.445 6.446 ... 6.458\n",
       "    ghi_min               (time, station) float64 40B 0.002086 ... 0.002086\n",
       "    ...                    ...\n",
       "    ghi_std               (time, station) float64 40B 0.0 3.4e-06 0.0 0.0 0.0\n",
       "    gti_std               (time, station) float64 40B 0.0 5.069e-06 ... 0.0 0.0\n",
       "    lat                   (time, station) float64 40B nan nan 51.39 51.39 51.39\n",
       "    lon                   (time, station) float64 40B nan nan 11.89 11.89 11.89\n",
       "    maintenance_flag_ghi  (maintenancetime, station) float32 4B 9.0\n",
       "    maintenance_flag_gti  (maintenancetime, station) float32 4B 7.0\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256666666667\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:04.065000\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M4.935S\n",
       "    time_coverage_resolution:  P0DT0H0M1.23375S
" ], "text/plain": [ " Size: 580B\n", "Dimensions: (station: 1, time: 5, maintenancetime: 1)\n", "Coordinates:\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n", "Data variables: (12/15)\n", " ghi (time, station) float64 40B 0.002086 ... 0.002086\n", " gti (time, station) float64 40B 0.001935 ... 0.001935\n", " ta (time, station) float64 40B 294.8 294.8 ... 294.8\n", " rh (time, station) float64 40B 0.6263 0.6266 ... 0.629\n", " battery_voltage (time, station) float64 40B 6.445 6.446 ... 6.458\n", " ghi_min (time, station) float64 40B 0.002086 ... 0.002086\n", " ... ...\n", " ghi_std (time, station) float64 40B 0.0 3.4e-06 0.0 0.0 0.0\n", " gti_std (time, station) float64 40B 0.0 5.069e-06 ... 0.0 0.0\n", " lat (time, station) float64 40B nan nan 51.39 51.39 51.39\n", " lon (time, station) float64 40B nan nan 11.89 11.89 11.89\n", " maintenance_flag_ghi (maintenancetime, station) float32 4B 9.0\n", " maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "\n", "# 7. Interpolate GPS coordinates to bin time\n", "ds_gps = ds_l1a.drop_dims(\"adctime\")\n", "ds_gps = ds_gps.drop_vars(['iadc'])\n", "\n", "\n", "# Decide whether geo coordinates should be averaged or not\n", "\n", "# if config['average_latlon']:\n", "ds_gps_avg = ds_gps.mean('gpstime',skipna=True)\n", "ds_l1b_avg = xr.merge((ds_l1b,ds_gps_avg),compat='no_conflicts')\n", "\n", "# else:\n", "ds_gps = ds_gps.interp(gpstime=ds_l1b.time,\n", " kwargs={\"bounds_error\":False, \"fill_value\":np.nan})\n", "ds_gps = ds_gps.drop_vars(\"gpstime\")\n", "\n", "ds_l1b = xr.merge((ds_l1b,ds_gps),compat='no_conflicts')\n", "\n", "ds_l1b" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:13.011890Z", "start_time": "2024-08-07T08:28:12.975961Z" }, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "# dsp = ds_l1b.sel(time=\"2019-07-15\")\n", "# plt.figure()\n", "# plt.plot(dsp.time, dsp.ghi)\n", "# plt.grid()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Add sun position\n", "Use ```trosat.sunpos``` to calculate sun position from time and lat,lon coordinates.\n" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:13.143009Z", "start_time": "2024-08-07T08:28:13.087128Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "szen (avg latlon): [42.51158528 42.51172144 42.51185788 42.5119946 42.51213159]\n", "szen: [ nan nan 42.51185936 42.5119952 42.51213125]\n", "sazi (avg latlon): [182.90675054 182.91284018 182.91892979 182.92501938 182.93110893]\n", "sazi: [ nan nan 182.91893638 182.92502264 182.93110846]\n", "Earth-Sun Distance: 1.009599726026352\n" ] } ], "source": [ "#|dropcode\n", "#|dropout\n", "# 8. Calc and add sun position\n", "\n", "szen, sazi = sp.sun_angles(\n", " time=ds_l1b.time.values[:,None],\n", " lat=ds_l1b.lat.values,\n", " lon=ds_l1b.lon.values\n", ")\n", "\n", "\n", "szen_avg, sazi_avg = sp.sun_angles(\n", " time=ds_l1b_avg.time.values[:,None],\n", " lat=ds_l1b_avg.lat.values,\n", " lon=ds_l1b_avg.lon.values\n", ")\n", "\n", "szen = szen.squeeze()\n", "sazi = sazi.squeeze()\n", "szen_avg = szen_avg.squeeze()\n", "sazi_avg = sazi_avg.squeeze()\n", "\n", "\n", "esd = np.mean(sp.earth_sun_distance(ds_l1b.time.values))\n", "\n", "print('szen (avg latlon):', szen_avg)\n", "print('szen:', szen)\n", "print('sazi (avg latlon):', sazi_avg)\n", "print('sazi:', sazi)\n", "print('Earth-Sun Distance:',esd)" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:13.352033Z", "start_time": "2024-08-07T08:28:13.157272Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 668B\n",
       "Dimensions:               (station: 1, time: 5, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * station               (station) float32 4B 1.0\n",
       "  * time                  (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n",
       "  * maintenancetime       (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
       "Data variables: (12/18)\n",
       "    ghi                   (time, station) float64 40B 0.002086 ... 0.002086\n",
       "    gti                   (time, station) float64 40B 0.001935 ... 0.001935\n",
       "    ta                    (time, station) float64 40B 294.8 294.8 ... 294.8\n",
       "    rh                    (time, station) float64 40B 0.6263 0.6266 ... 0.629\n",
       "    battery_voltage       (time, station) float64 40B 6.445 6.446 ... 6.458\n",
       "    ghi_min               (time, station) float64 40B 0.002086 ... 0.002086\n",
       "    ...                    ...\n",
       "    lon                   (time, station) float64 40B nan nan 11.89 11.89 11.89\n",
       "    maintenance_flag_ghi  (maintenancetime, station) float32 4B 9.0\n",
       "    maintenance_flag_gti  (maintenancetime, station) float32 4B 7.0\n",
       "    szen                  (time, station) float64 40B nan nan 42.51 42.51 42.51\n",
       "    sazi                  (time, station) float64 40B nan nan 182.9 182.9 182.9\n",
       "    esd                   (station) float64 8B 1.01\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256666666667\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:04.065000\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M4.935S\n",
       "    time_coverage_resolution:  P0DT0H0M1.23375S
" ], "text/plain": [ " Size: 668B\n", "Dimensions: (station: 1, time: 5, maintenancetime: 1)\n", "Coordinates:\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n", "Data variables: (12/18)\n", " ghi (time, station) float64 40B 0.002086 ... 0.002086\n", " gti (time, station) float64 40B 0.001935 ... 0.001935\n", " ta (time, station) float64 40B 294.8 294.8 ... 294.8\n", " rh (time, station) float64 40B 0.6263 0.6266 ... 0.629\n", " battery_voltage (time, station) float64 40B 6.445 6.446 ... 6.458\n", " ghi_min (time, station) float64 40B 0.002086 ... 0.002086\n", " ... ...\n", " lon (time, station) float64 40B nan nan 11.89 11.89 11.89\n", " maintenance_flag_ghi (maintenancetime, station) float32 4B 9.0\n", " maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n", " szen (time, station) float64 40B nan nan 42.51 42.51 42.51\n", " sazi (time, station) float64 40B nan nan 182.9 182.9 182.9\n", " esd (station) float64 8B 1.01\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "ds_l1b = ds_l1b.assign(\n", " {\n", " \"szen\": ((\"time\", \"station\"), szen[:,None]),\n", " \"sazi\": ((\"time\", \"station\"), sazi[:,None]),\n", " \"esd\": (\"station\", [esd])\n", " }\n", ")\n", "for key in ['szen', 'sazi','esd']:\n", " ds_l1b[key].attrs.update(vattrs[key])\n", " # ds_l1b[key].encoding.update(vencode[key])\n", "ds_l1b" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### **(Not implemented)** Alignment detection\n", "Detect the alignment of the instruments assuming perfect alignment at the beginning of the maintenance period. Refine GTI *hangle* and *vangle* if available. " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Calibrate radiation flux\n", "\n", "Calibration and correction of measured voltage (U) is done via the following equations:\n", "\n", "$ \\mathrm{GHI} = \\mathrm{U} * \\mathrm{C}_\\mathrm{a} * \\mathrm{C}_\\mathrm{c}(\\mu_a) * \\frac{\\mu_0}{\\mu_a}$,\n", "\n", "$ \\mathrm{GTI} = \\mathrm{U} * \\mathrm{C}_\\mathrm{a} * \\mathrm{C}_\\mathrm{c}(\\mu_a)$,\n", "\n", "with the absolute calibration factor $\\mathrm{C}_\\mathrm{a}$ $\\left(\\frac{W}{m^2}V^{-1}\\right)$ and the cosine correction factor $\\mathrm{C}_\\mathrm{c}$ which depends on the cosine of the apparent solar zenith angle ($\\mu_a$). The fraction $\\frac{\\mu_0}{\\mu_a}$ is applied as a factor to correct for misalignment (neglecting diffuse radiation) ([Boers et al. (1998)](http://dx.doi.org/10.1029/98JD01431)).\n", "\n", "Therefore, the full calibration including cosine and misalignment correction can only be applied if the apparent zenith angle is known. For GHI we apply the full calibration assuming $\\mu_a = \\mu_0$. For GTI, only the absolute calibration is added. GTI cosine correction is applied only if both *hangle* and *vangle* are known. " ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:13.681570Z", "start_time": "2024-08-07T08:28:13.361487Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Meta Lookup:\n", ">> Box = 1.0\n", ">> serial(s) = ['S12128.001', 'S12137.049']\n", ">> calibration factor(s) = [7.73, 6.98]\n", ">> cosine correction factor = 1.45 - 3.04·x + 5.59·x² - 3.01·x³,\n", ">> with x = cos(apparent solar zenith angle)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 668B\n",
       "Dimensions:               (station: 1, time: 5, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * station               (station) float32 4B 1.0\n",
       "  * time                  (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n",
       "  * maintenancetime       (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
       "Data variables: (12/18)\n",
       "    ghi                   (time, station) float64 40B 269.9 269.7 ... 280.9\n",
       "    gti                   (time, station) float64 40B 277.3 277.3 ... 288.6\n",
       "    ta                    (time, station) float64 40B 294.8 294.8 ... 294.8\n",
       "    rh                    (time, station) float64 40B 0.6263 0.6266 ... 0.629\n",
       "    battery_voltage       (time, station) float64 40B 6.445 6.446 ... 6.458\n",
       "    ghi_min               (time, station) float64 40B 269.9 268.5 ... 280.9\n",
       "    ...                    ...\n",
       "    lon                   (time, station) float64 40B nan nan 11.89 11.89 11.89\n",
       "    maintenance_flag_ghi  (maintenancetime, station) float32 4B 9.0\n",
       "    maintenance_flag_gti  (maintenancetime, station) float32 4B 7.0\n",
       "    szen                  (time, station) float64 40B nan nan 42.51 42.51 42.51\n",
       "    sazi                  (time, station) float64 40B nan nan 182.9 182.9 182.9\n",
       "    esd                   (station) float64 8B 1.01\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256666666667\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:04.065000\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M4.935S\n",
       "    time_coverage_resolution:  P0DT0H0M1.23375S
" ], "text/plain": [ " Size: 668B\n", "Dimensions: (station: 1, time: 5, maintenancetime: 1)\n", "Coordinates:\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n", "Data variables: (12/18)\n", " ghi (time, station) float64 40B 269.9 269.7 ... 280.9\n", " gti (time, station) float64 40B 277.3 277.3 ... 288.6\n", " ta (time, station) float64 40B 294.8 294.8 ... 294.8\n", " rh (time, station) float64 40B 0.6263 0.6266 ... 0.629\n", " battery_voltage (time, station) float64 40B 6.445 6.446 ... 6.458\n", " ghi_min (time, station) float64 40B 269.9 268.5 ... 280.9\n", " ... ...\n", " lon (time, station) float64 40B nan nan 11.89 11.89 11.89\n", " maintenance_flag_ghi (maintenancetime, station) float32 4B 9.0\n", " maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n", " szen (time, station) float64 40B nan nan 42.51 42.51 42.51\n", " sazi (time, station) float64 40B nan nan 182.9 182.9 182.9\n", " esd (station) float64 8B 1.01\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# 5. rad flux calibration\n", "box = ds_l1b.station.values[0]\n", "boxnumber, serial, cfac, CCcoef = pyrnet.pyrnet.meta_lookup(\n", " ds_l1b.time.values[0],\n", " box=box,\n", " cfile=config['file_calibration'],\n", " mapfile=config['file_mapping'],\n", ")\n", "\n", "print(f\"Meta Lookup:\")\n", "print(f\">> Box = {box}\")\n", "print(f\">> serial(s) = {serial}\")\n", "print(f\">> calibration factor(s) = {cfac}\")\n", "print(f\">> cosine correction factor = {np.polynomial.polynomial.Polynomial(CCcoef)},\\n>> with x = cos(apparent solar zenith angle)\")\n", "\n", "mu0 = np.cos(np.deg2rad(ds_l1b.szen.values))\n", "\n", "# calibrate radiation flux with gain=300\n", "for i, radflx in enumerate(config['radflux_varname']):\n", " # all radflux related variables (including _ variables)\n", " radflx_vars = [var for var in ds_l1b if var.startswith(radflx)]\n", " if cfac[i] is None:\n", " # drop if calibration/instrument don't exist (probably secondary pyranometer).\n", " ds_l1b = ds_l1b.drop_vars(radflx_vars)\n", " continue\n", " \n", " # calc apparent zenith angle if possible\n", " mua = mu0.copy()\n", " if \"vangle\" in ds[radflx].attrs:\n", " vangle = pyrnet.utils.make_iter(ds[radflx].attrs[\"vangle\"])\n", " hangle = pyrnet.utils.make_iter(ds[radflx].attrs[\"hangle\"])\n", " mua = pyrnet.utils.calc_apparent_coszen(\n", " pitch=vangle,\n", " yaw=hangle,\n", " zen=ds_l1b.szen.values,\n", " azi=ds_l1b.sazi.values\n", " )\n", " mua[mua<=0] = np.nan\n", " mask_mua = ~np.isnan(mua)\n", " Ca = 1e6/cfac[i]\n", " Cc = np.polynomial.polynomial.polyval(mua, c=CCcoef)\n", " Cmu = mu0/mua\n", " # apply to all variables\n", " for var in radflx_vars:\n", " calib_func = \"flux (W m-2) = flux (V) * Cabsolute (W m-2 V-1)\"\n", " C = np.ones(mu0.shape)*Ca\n", " if radflx == \"gti\":\n", " C[mask_mua] *= Cc[mask_mua]\n", " calib_func += \"\" if np.all(np.isnan(mua)) else \" * Ccoscorr(mua)\"\n", " else:\n", " C[mask_mua] *= Cc[mask_mua] * Cmu[mask_mua]\n", " calib_func += \" * Ccoscorr(mua)\"# * mu0/mua\" (not implemented)\n", " ds_l1b[var].values = ds_l1b[var].values*C\n", "\n", " ds_l1b[var].attrs['units'] = \"W m-2\",\n", " ds_l1b[var].attrs.update({\n", " \"units\": \"W m-2\",\n", " \"serial\": serial[i],\n", " \"calibration_Cabsolute\": Ca,\n", " \"calibration_Ccoscorr\": str(np.polynomial.polynomial.Polynomial(CCcoef)),\n", " \"calibration_function\": calib_func\n", " })\n", "ds_l1b" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Add automatic quality flags\n", "Add BSRN recommended limit checks and intercompare checks if multiple pyranometers are in the dataset." ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:14.253424Z", "start_time": "2024-08-07T08:28:13.693268Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 678B\n",
       "Dimensions:               (station: 1, time: 5, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * station               (station) float32 4B 1.0\n",
       "  * time                  (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n",
       "  * maintenancetime       (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
       "Data variables: (12/20)\n",
       "    ghi                   (time, station) float64 40B 269.9 269.7 ... 280.9\n",
       "    gti                   (time, station) float64 40B 277.3 277.3 ... 288.6\n",
       "    ta                    (time, station) float64 40B 294.8 294.8 ... 294.8\n",
       "    rh                    (time, station) float64 40B 0.6263 0.6266 ... 0.629\n",
       "    battery_voltage       (time, station) float64 40B 6.445 6.446 ... 6.458\n",
       "    ghi_min               (time, station) float64 40B 269.9 268.5 ... 280.9\n",
       "    ...                    ...\n",
       "    maintenance_flag_gti  (maintenancetime, station) float32 4B 7.0\n",
       "    szen                  (time, station) float64 40B nan nan 42.51 42.51 42.51\n",
       "    sazi                  (time, station) float64 40B nan nan 182.9 182.9 182.9\n",
       "    esd                   (station) float64 8B 1.01\n",
       "    qc_flag_ghi           (time, station) uint8 5B 0 0 0 0 0\n",
       "    qc_flag_gti           (time, station) uint8 5B 0 0 0 0 0\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256666666667\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:04.065000\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M4.935S\n",
       "    time_coverage_resolution:  P0DT0H0M1.23375S
" ], "text/plain": [ " Size: 678B\n", "Dimensions: (station: 1, time: 5, maintenancetime: 1)\n", "Coordinates:\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n", "Data variables: (12/20)\n", " ghi (time, station) float64 40B 269.9 269.7 ... 280.9\n", " gti (time, station) float64 40B 277.3 277.3 ... 288.6\n", " ta (time, station) float64 40B 294.8 294.8 ... 294.8\n", " rh (time, station) float64 40B 0.6263 0.6266 ... 0.629\n", " battery_voltage (time, station) float64 40B 6.445 6.446 ... 6.458\n", " ghi_min (time, station) float64 40B 269.9 268.5 ... 280.9\n", " ... ...\n", " maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n", " szen (time, station) float64 40B nan nan 42.51 42.51 42.51\n", " sazi (time, station) float64 40B nan nan 182.9 182.9 182.9\n", " esd (station) float64 8B 1.01\n", " qc_flag_ghi (time, station) uint8 5B 0 0 0 0 0\n", " qc_flag_gti (time, station) uint8 5B 0 0 0 0 0\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256666666667\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:04.065000\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M4.935S\n", " time_coverage_resolution: P0DT0H0M1.23375S" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "ds_l1b = pyrnet.qcrad.add_qc_flags(ds_l1b, config[\"radflux_varname\"])\n", "ds_l1b" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Update variables and global attributes and encoding" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:14.535285Z", "start_time": "2024-08-07T08:28:14.290796Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 678B\n",
       "Dimensions:               (station: 1, time: 5, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * station               (station) float32 4B 1.0\n",
       "  * time                  (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n",
       "  * maintenancetime       (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
       "Data variables: (12/20)\n",
       "    ghi                   (time, station) float64 40B 269.9 269.7 ... 280.9\n",
       "    gti                   (time, station) float64 40B 277.3 277.3 ... 288.6\n",
       "    ta                    (time, station) float64 40B 294.8 294.8 ... 294.8\n",
       "    rh                    (time, station) float64 40B 0.6263 0.6266 ... 0.629\n",
       "    battery_voltage       (time, station) float64 40B 6.445 6.446 ... 6.458\n",
       "    ghi_min               (time, station) float64 40B 269.9 268.5 ... 280.9\n",
       "    ...                    ...\n",
       "    maintenance_flag_gti  (maintenancetime, station) float32 4B 7.0\n",
       "    szen                  (time, station) float64 40B nan nan 42.51 42.51 42.51\n",
       "    sazi                  (time, station) float64 40B nan nan 182.9 182.9 182.9\n",
       "    esd                   (station) float64 8B 1.01\n",
       "    qc_flag_ghi           (time, station) uint8 5B 0 0 0 0 0\n",
       "    qc_flag_gti           (time, station) uint8 5B 0 0 0 0 0\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885256558333333\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:03\n",
       "    time_coverage_end:         2022-08-30T11:21:07\n",
       "    time_coverage_duration:    P0DT0H0M4S\n",
       "    time_coverage_resolution:  P0DT0H0M1S
" ], "text/plain": [ " Size: 678B\n", "Dimensions: (station: 1, time: 5, maintenancetime: 1)\n", "Coordinates:\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 40B 2022-08-30T11:21:03 ... 2...\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n", "Data variables: (12/20)\n", " ghi (time, station) float64 40B 269.9 269.7 ... 280.9\n", " gti (time, station) float64 40B 277.3 277.3 ... 288.6\n", " ta (time, station) float64 40B 294.8 294.8 ... 294.8\n", " rh (time, station) float64 40B 0.6263 0.6266 ... 0.629\n", " battery_voltage (time, station) float64 40B 6.445 6.446 ... 6.458\n", " ghi_min (time, station) float64 40B 269.9 268.5 ... 280.9\n", " ... ...\n", " maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n", " szen (time, station) float64 40B nan nan 42.51 42.51 42.51\n", " sazi (time, station) float64 40B nan nan 182.9 182.9 182.9\n", " esd (station) float64 8B 1.01\n", " qc_flag_ghi (time, station) uint8 5B 0 0 0 0 0\n", " qc_flag_gti (time, station) uint8 5B 0 0 0 0 0\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885256558333333\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:03\n", " time_coverage_end: 2022-08-30T11:21:07\n", " time_coverage_duration: P0DT0H0M4S\n", " time_coverage_resolution: P0DT0H0M1S" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "# add global coverage attributes\n", "ds_l1b = update_coverage_meta(ds_l1b, timevar=\"time\")\n", "\n", "ds_l1b.attrs[\"processing_level\"] = 'l1b'\n", "now = pd.to_datetime(np.datetime64(\"now\"))\n", "ds_l1b.attrs[\"history\"] = ds_l1b.history + f\"{now.isoformat()}: Generated level l1b by pyrnet version {pyrnet_version}; \"\n", "\n", "ds_l1b" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:14.561074Z", "start_time": "2024-08-07T08:28:14.544523Z" }, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "# dsp = ds_l1b.sel(time=\"2019-07-15\")\n", "# plt.figure()\n", "# plt.plot(dsp.time, dsp.ghi)\n", "# plt.grid()" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:14.599076Z", "start_time": "2024-08-07T08:28:14.582824Z" }, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "# ds_l1b.to_netcdf(\"../../testnb/to_l1b_output.nc\",\n", "# encoding={'time':{'dtype':'float64'}})" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:14.634812Z", "start_time": "2024-08-07T08:28:14.616229Z" }, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "# dsp = xr.load_dataset(\"../../testnb/to_l1b_output.nc\").sel(time=\"2019-07-15\")\n", "# plt.figure()\n", "# plt.plot(dsp.time, dsp.ghi)\n", "# plt.grid()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## to_l1b function" ] }, { "cell_type": "code", "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:14.845812Z", "start_time": "2024-08-07T08:28:14.655987Z" }, "tags": [ "hide-input" ] }, "outputs": [], "source": [ "#|export\n", "#|dropcode\n", "def to_l1b(\n", " fname: str,\n", " *,\n", " config: dict | None = None,\n", " global_attrs: dict | None = None,\n", " check_adc_sync: bool = True\n", ") -> xr.Dataset|None:\n", "\n", " config = get_config(config)\n", " gattrs, vattrs, vencode = get_cfmeta(config)\n", "\n", " if global_attrs is not None:\n", " gattrs.update(global_attrs)\n", "\n", " ######################################################################################\n", " ## Load l1a data\n", " ds_l1a = xr.open_dataset(fname)\n", " # check correct file\n", " if ds_l1a.processing_level != \"l1a\":\n", " logger.warning(f\"{fname} is not a l1a file. Skip.\")\n", " return None\n", "\n", " ######################################################################################\n", " ## Sync GPS to ADC time\n", " adctime = pyrnet.logger.sync_adc_time(\n", " adctime = ds_l1a.adctime.values,\n", " gpstime = ds_l1a.gpstime.values,\n", " iadc = ds_l1a.iadc.squeeze().values.astype(int),\n", " check_results = check_adc_sync\n", " )\n", " \n", " if adctime is None:\n", " logger.warning(f\"Could not fit GPS to ADC time for file {fname}. Skip.\")\n", " return None\n", "\n", " ######################################################################################\n", " ## Create new dataset (l1b)\n", " ds_l1b = ds_l1a.drop_dims('gpstime')\n", " ds_l1b = ds_l1b.drop_vars(['maintenance_flag_ghi','maintenance_flag_gti']) # keep only time dependent variables\n", " ds_l1b = ds_l1b.assign({'time': ('adctime', adctime)})\n", " ds_l1b = ds_l1b.swap_dims({\"adctime\":\"time\"})\n", " ds_l1b = ds_l1b.drop_vars(\"adctime\")\n", "\n", " ds_l1b[\"time\"].encoding.update({\n", " \"dtype\": 'float64',\n", " \"units\": f\"seconds since {np.datetime_as_string(ds_l1b.time.data[0], unit='D')}T00:00Z\",\n", " })\n", " logger.info(f\"Dataset time coverage before strip: {ds_l1b.time.values[0]} - {ds_l1b.time.values[-1]}\")\n", "\n", " ######################################################################################\n", " ## Drop first and last minutes of data to avoid bad data due to maintenance\n", " stripminutes = np.timedelta64(int(config['stripminutes']), 'm')\n", " if (ds_l1b.time.values[0] + 3*stripminutes) > ds_l1b.time.values[-1]:\n", " logger.warning(f\"{fname} has not enough data. Skip.\")\n", " return None\n", "\n", " ds_l1b = ds_l1b.isel(time=ds_l1b.time>ds_l1b.time.values[0] + stripminutes)\n", " ds_l1b = ds_l1b.isel(time=ds_l1b.time> Box={box}\")\n", " logger.info(f\">> serial(s)={serial}\")\n", " logger.info(f\">> calibration factor(s)={cfac}\")\n", "\n", " mu0 = np.cos(np.deg2rad(ds_l1b.szen.values))\n", " \n", " # calibrate radiation flux with gain=300\n", " for i, radflx in enumerate(config['radflux_varname']):\n", " # all radflux related variables (including _ variables)\n", " radflx_vars = [var for var in ds_l1b if var.startswith(radflx)]\n", " if cfac[i] is None:\n", " # drop if calibration/instrument don't exist (probably secondary pyranometer).\n", " ds_l1b = ds_l1b.drop_vars(radflx_vars)\n", " continue\n", " \n", " # calc apparent zenith angle if possible\n", " mua = mu0.copy()\n", " if \"vangle\" in ds_l1b[radflx].attrs:\n", " vangle = pyrnet.utils.make_iter(ds_l1b[radflx].attrs[\"vangle\"])\n", " hangle = pyrnet.utils.make_iter(ds_l1b[radflx].attrs[\"hangle\"])\n", " mua = pyrnet.utils.calc_apparent_coszen(\n", " pitch=vangle,\n", " yaw=hangle,\n", " zen=ds_l1b.szen.values,\n", " azi=ds_l1b.sazi.values\n", " )\n", " mua[mua<=0] = np.nan\n", " mask_mua = ~np.isnan(mua)\n", " Ca = 1e6/cfac[i]\n", " Cc = np.polynomial.polynomial.polyval(mua, c=CCcoef)\n", " Cmu = mu0/mua\n", " # apply to all variables\n", " for var in radflx_vars:\n", " calib_func = \"flux (W m-2) = flux (V) * Cabsolute (W m-2 V-1)\"\n", " C = np.ones(mu0.shape)*Ca\n", " if radflx == \"gti\":\n", " C[mask_mua] *= Cc[mask_mua]\n", " calib_func += \"\" if np.all(np.isnan(mua)) else \" * Ccoscorr(mua)\"\n", " else:\n", " C[mask_mua] *= Cc[mask_mua] * Cmu[mask_mua]\n", " calib_func += \" * Ccoscorr(mua)\" # * mu0/mua\" (not implemented)\n", " ds_l1b[var].values = ds_l1b[var].values*C\n", " \n", " ds_l1b[var].attrs['units'] = \"W m-2\",\n", " ds_l1b[var].attrs.update({\n", " \"units\": \"W m-2\",\n", " \"serial\": serial[i],\n", " \"calibration_Cabsolute\": Ca,\n", " \"calibration_Ccoscorr\": str(np.polynomial.polynomial.Polynomial(CCcoef)),\n", " \"calibration_function\": calib_func\n", " })\n", "\n", " ###################################################################################### \n", " ## add quality flags\n", " ds_l1b = pyrnet.qcrad.add_qc_flags(ds_l1b, config[\"radflux_varname\"])\n", "\n", " ######################################################################################\n", " ## Update variables, global attributes and encoding\n", " #add global coverage attributes\n", " ds_l1b = update_coverage_meta(ds_l1b, timevar=\"time\")\n", " ds_l1b.attrs[\"processing_level\"] = 'l1b'\n", " ds_l1b.attrs[\"product_version\"] = pyrnet_version\n", " now = pd.to_datetime(np.datetime64(\"now\"))\n", " ds_l1b.attrs[\"history\"] = ds_l1b.history + f\"{now.isoformat()}: Generated level l1b by pyrnet version {pyrnet_version}; \"\n", "\n", " # update encoding\n", " ds_l1b = add_encoding(ds_l1b, vencode=vencode)\n", "\n", " return ds_l1b" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Test to_l1b function" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:14.925756Z", "start_time": "2024-08-07T08:28:14.907869Z" }, "tags": [ "remove-cell", "hide-output" ] }, "outputs": [], "source": [ "#|hide\n", "#|dropout\n", "# \n", "# # process raw data \n", "# fn_report = \"../../example_data/results-survey224783.csv\"\n", "# fn_cfmeta = os.path.join(importlib.resources.files(\"pyrnet\"), \"share/pyrnet_cfmeta.c01.json\")\n", "# \n", "# \n", "# # parse report\n", "# df_report = pyrnet.reports.get_responses(fn=\"../../example_data/results-survey224783.csv\")\n", "# report = pyrnet.reports.parse_report(df_report,\n", "# date_of_maintenance=np.datetime64(\"2023-05-08T12:00\"))\n", "# \n", "# fn_datas = [\n", "# \"../../example_data/raw/Pyr9_000.bin\",\n", "# \"../../example_data/raw/Pyr1_000.bin\", \n", "# \"../../example_data/raw/Pyr14_000.bin\"\n", "# ]\n", "# for i,fn_data in enumerate(fn_datas):\n", "# # read logger file to xarray\n", "# ds = to_l1a(\n", "# fname=fn_data,\n", "# station=1, # actually test data is from station 9, but test reports are for station 1 and 2 only\n", "# # bins=86400, # seconds resolution\n", "# report=report,\n", "# config={\"file_cfmeta\": fn_cfmeta, \"stripminutes\": 0},\n", "# global_attrs={\"TESTNOTE\": \"This is a test note.\"}\n", "# )\n", "# \n", "# print(\"Processing fname:\")\n", "# print(get_fname(ds, freq=\"10Hz\", timevar=\"gpstime\", config=None))\n", "# \n", "# if i==0:\n", "# ds.to_netcdf(\"../../example_data/to_l1a_output.nc\")\n", "# else:\n", "# ds.to_netcdf(f\"../../example_data/to_l1a_output_{i}.nc\")" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:17.661023Z", "start_time": "2024-08-07T08:28:14.940119Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Processing fname:\n", "2022-08-30_P0DT0H0M8S_pyrnet_test_s001l1bf1s.c01.nc\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 1kB\n",
       "Dimensions:               (station: 1, time: 9, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * station               (station) float32 4B 1.0\n",
       "  * time                  (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n",
       "  * maintenancetime       (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
       "Data variables: (12/20)\n",
       "    ghi                   (time, station) float64 72B 280.9 280.9 ... 280.9\n",
       "    gti                   (time, station) float64 72B 289.0 289.0 ... 289.4\n",
       "    ta                    (time, station) float64 72B 294.8 294.7 ... 294.8\n",
       "    rh                    (time, station) float64 72B 0.6253 0.6248 ... 0.629\n",
       "    battery_voltage       (time, station) float64 72B 6.443 6.445 ... 6.465\n",
       "    gti_min               (time, station) float64 72B 288.6 288.6 ... 288.6\n",
       "    ...                    ...\n",
       "    maintenance_flag_gti  (maintenancetime, station) float32 4B 7.0\n",
       "    szen                  (time, station) float64 72B 42.51 42.51 ... 42.51\n",
       "    sazi                  (time, station) float64 72B 182.9 182.9 ... 182.9\n",
       "    esd                   (station) float64 8B 1.01\n",
       "    qc_flag_ghi           (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
       "    qc_flag_gti           (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
       "Attributes: (12/31)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Generated level l1a  by p...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_max:        11.885252\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:01\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M8S\n",
       "    time_coverage_resolution:  P0DT0H0M1S
" ], "text/plain": [ " Size: 1kB\n", "Dimensions: (station: 1, time: 9, maintenancetime: 1)\n", "Coordinates:\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n", "Data variables: (12/20)\n", " ghi (time, station) float64 72B 280.9 280.9 ... 280.9\n", " gti (time, station) float64 72B 289.0 289.0 ... 289.4\n", " ta (time, station) float64 72B 294.8 294.7 ... 294.8\n", " rh (time, station) float64 72B 0.6253 0.6248 ... 0.629\n", " battery_voltage (time, station) float64 72B 6.443 6.445 ... 6.465\n", " gti_min (time, station) float64 72B 288.6 288.6 ... 288.6\n", " ... ...\n", " maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n", " szen (time, station) float64 72B 42.51 42.51 ... 42.51\n", " sazi (time, station) float64 72B 182.9 182.9 ... 182.9\n", " esd (station) float64 8B 1.01\n", " qc_flag_ghi (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n", " qc_flag_gti (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n", "Attributes: (12/31)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Generated level l1a by p...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_max: 11.885252\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:01\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M8S\n", " time_coverage_resolution: P0DT0H0M1S" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropcode\n", "#|dropout\n", "fname = \"../../example_data/to_l1a_output.nc\"\n", "config = {\n", " \"l1bfreq\":\"1s\",\n", " \"campaign\": \"test\",\n", " \"stripminutes\":0,\n", " \"average_latlon\":True,\n", " \"l1b_resample_stats\": [\"min\", \"max\", \"std\"],\n", " \"radflux_varname\": [\"ghi\",\"gti\"]\n", "}\n", "\n", "ds_l1b = to_l1b(fname=fname, config=config, check_adc_sync=False)\n", "\n", "print(\"Processing fname:\")\n", "print(get_fname(ds_l1b, freq='1s', timevar=\"time\", config=config))\n", "\n", "ds_l1b.to_netcdf(\"../../example_data/to_l1b_output.nc\")\n", "ds_l1b" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:21.469777Z", "start_time": "2024-08-07T08:28:17.670072Z" } }, "outputs": [], "source": [ "# import cfchecker.cfchecks\n", "# init = cfchecker.cfchecks.CFChecker()\n", "# res = init.checker(\"../../example_data/to_l1b_output.nc\")" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:21.807809Z", "start_time": "2024-08-07T08:28:21.475137Z" }, "tags": [ "hide-output" ] }, "outputs": [ { "data": { "text/plain": [ "\n", "root group (NETCDF4 data model, file format HDF5):\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observational data set\n", " history: 2025-10-16T11:47:26: Generated level l1a by pyrnet version 1.0.0+1.g74c8a89; 2025-10-16T11:47:26: Generated level l1b by pyrnet version 1.0.0+1.g74c8a89; \n", " institution: Leibniz Institute for Tropospheric Research (TROPOS)\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " Department: Remote Sensing of Atmospheric Processes\n", " Department_team: Clouds, Aerosol and Radiation\n", " Address: Permoser Str. 15, 04318 Leipzig, Germany\n", " Contact_person: Andreas Macke and the clouds, aerosol and radiation team of the remote sensing department, mailto:andreas.macke@tropos.de\n", " Contributor_name: \n", " Contributor_role: \n", " Authors_software: Hartwig Deneke, Jonas Witthuhn, mailto:deneke@tropos.de\n", " Creator_name: \n", " Project: \n", " Standard_name_vocabulary: CF Standard Name Table v81\n", " License: CC-BY-SA 3.0\n", " TESTNOTE: This is a test note.\n", " processing_level: l1b\n", " product_version: 1.0.0+1.g74c8a89\n", " date_created: 2025-10-16T11:47:26\n", " geospatial_lat_min: 51.390210333333336\n", " geospatial_lat_max: 51.390210333333336\n", " geospatial_lat_units: degN\n", " geospatial_lon_min: 11.885252\n", " geospatial_lon_max: 11.885252\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:01\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M8S\n", " time_coverage_resolution: P0DT0H0M1S\n", " dimensions(sizes): station(1), time(9), maintenancetime(1)\n", " variables(dimensions): float32 station(station), float64 time(time), uint16 ghi(time, station), uint16 gti(time, station), uint16 ta(time, station), uint16 rh(time, station), uint16 battery_voltage(time, station), float64 maintenancetime(maintenancetime), uint16 gti_min(time, station), uint16 ghi_min(time, station), uint16 gti_max(time, station), uint16 ghi_max(time, station), uint16 gti_std(time, station), uint16 ghi_std(time, station), float64 lat(station), float64 lon(station), uint8 maintenance_flag_ghi(maintenancetime, station), uint8 maintenance_flag_gti(maintenancetime, station), uint16 szen(time, station), uint16 sazi(time, station), float64 esd(station), uint8 qc_flag_ghi(time, station), uint8 qc_flag_gti(time, station)\n", " groups: " ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropout\n", "import netCDF4\n", "netCDF4.Dataset(\"../../example_data/to_l1b_output.nc\",'r')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:21.828237Z", "start_time": "2024-08-07T08:28:21.815568Z" } }, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Merging\n", "Merging datasets with xarray default merge overwrites or drops certain variable attributes (e.g., serial number, calibration factor). The methods here are specialized to merge PyrNet datasets and attributes. " ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:21.926727Z", "start_time": "2024-08-07T08:28:21.840568Z" } }, "outputs": [], "source": [ "# make 3 artificial datasets for testing\n", "# original ds_l1b\n", "ds1 = ds_l1b.copy()\n", "ds1.attrs[\"site\"] = \"testsite1\"\n", "# ds_l1b but new times\n", "ds2 = ds1.copy()\n", "ds2 = ds2.assign_coords({\n", " \"time\": (\"time\", ds2.time.values+(ds2.time.values[-1]-ds2.time.values[0]))\n", "})\n", "# ds_l1b, same time but other station\n", "ds3 = ds1.copy()\n", "ds3 = ds3.assign_coords({\n", " \"station\": (\"station\", [2])\n", "})\n", "ds3.attrs[\"site\"] = \"testsite2\"\n", "for var in ds3:\n", " if \"ghi\" in var:\n", " ds3[var].attrs[\"serial\"] = \"test_ghi_ds3\"\n", " if \"gti\" in var:\n", " ds3[var].attrs[\"serial\"] = \"test_gti_ds3\"\n", "\n", "dslist = [ds1, ds3, ds2]\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Sort by station coordinates\n", "Before merging, sort them by station" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:21.967797Z", "start_time": "2024-08-07T08:28:21.936736Z" }, "tags": [ "hide-input" ] }, "outputs": [], "source": [ "#|export\n", "#|dropcode\n", "def _sort_by_station(dslist):\n", " # sort dslist for first station\n", " station0 = []\n", " for i in range(len(dslist)):\n", " station0.append(int(dslist[i][\"station\"].values[0]))\n", " isort = np.argsort(station0).ravel()\n", " dslist = [dslist[i] for i in isort]\n", " return dslist\n" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:22.145055Z", "start_time": "2024-08-07T08:28:21.977116Z" }, "tags": [ "hide-output" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[1.]\n", "[1.]\n", "[2]\n" ] }, { "data": { "text/plain": [ "[None, None, None]" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropout\n", "dslist = _sort_by_station(dslist)\n", "[print(dslist[i].station.values) for i in range(len(dslist))]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Merge global and variable attributes\n", "Some attributes are tied to a specific statin (e.g., calibration factor, serial number). These attributes are converted to a list and merged with respect to the station dimension." ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:22.379997Z", "start_time": "2024-08-07T08:28:22.189818Z" }, "tags": [ "hide-input" ] }, "outputs": [], "source": [ "#|export\n", "#|dropcode\n", "def _merge_gattrs_by_station(dslist, merge_gattrs):\n", " # merge variable attrs:\n", " merge_gattrs_fill_value = [merge_gattrs[key] for key in merge_gattrs] \n", " merge_gattrs = [key for key in merge_gattrs]\n", " \n", " merged_gattrs = {}\n", " gattrs_idx = {}\n", " for i in range(len(dslist)):\n", " dst = dslist[i]\n", " for j,attr in enumerate(merge_gattrs):\n", " if attr not in dst.attrs:\n", " fill_value = dst.station.size * [merge_gattrs_fill_value[j]]\n", " dst.attrs.update({\n", " attr: fill_value\n", " })\n", " # save station index, as attributes are related to station dimension\n", " attridx = list(dst.station.values.astype(int))\n", " attrval = list(pyrnet.utils.make_iter(dst.attrs[attr]))\n", " # merge attributes, overwrite values of same station\n", " if attr in merged_gattrs:\n", " mattrval = merged_gattrs[attr] + attrval\n", " mattridx = gattrs_idx[attr] + attridx\n", " _,idx = np.unique(mattridx, return_index=True)\n", " attridx = [mattridx[i] for i in idx]\n", " attrval = [mattrval[i] for i in idx]\n", " gattrs_idx = assoc_in(\n", " gattrs_idx, [attr], attridx\n", " )\n", " merged_gattrs = assoc_in(\n", " merged_gattrs, [attr], attrval\n", " )\n", " return merged_gattrs\n", "\n", "def _merge_vattrs_by_station(dslist, merge_attrs):\n", " # add missing gti variables\n", " for i in range(len(dslist)):\n", " dst = dslist[i].copy()\n", " if \"gti\" in dst:\n", " continue\n", " ghi_vars = [var for var in dst if \"ghi\" in var]\n", " gti_vars = [var.replace(\"ghi\",\"gti\") for var in ghi_vars]\n", " for ghi_var, gti_var in zip(ghi_vars,gti_vars):\n", " dst = dst.assign({\n", " gti_var: (dst[ghi_var].dims, np.full(dst[ghi_var].shape, np.nan))\n", " })\n", " for attr in merge_attrs:\n", " skip = True\n", " for apply_to in merge_attrs[attr][\"apply_to\"]:\n", " if gti_var.startswith(apply_to):\n", " skip = False\n", " if skip:\n", " continue\n", " fill_value = dst.station.size * [merge_attrs[attr][\"fill_value\"]]\n", " dst[gti_var].attrs.update({\n", " attr: fill_value\n", " })\n", " dslist[i] = dst\n", " \n", " merged_attrs = {}\n", " mattrs_idx = {}\n", " for i in range(len(dslist)):\n", " dst = dslist[i]\n", " for var in dst:\n", " for attr in merge_attrs:\n", " skip = True\n", " for apply_to in merge_attrs[attr][\"apply_to\"]:\n", " if var.startswith(apply_to):\n", " skip = False\n", " if skip:\n", " continue\n", " if attr not in dst[var].attrs:\n", " fill_value = dst.station.size * [merge_attrs[attr][\"fill_value\"]]\n", " dst[var].attrs.update({\n", " attr: fill_value\n", " })\n", " # save station index, as attributes are related to station dimension\n", " attridx = list(dst.station.values.astype(int))\n", " attrval = list(pyrnet.utils.make_iter(dst[var].attrs[attr]))\n", "\n", " # merge attributes, overwrite values of same station\n", " if var not in merged_attrs:\n", " merged_attrs.update({var:{}})\n", " mattrs_idx.update({var:{}})\n", " if attr in merged_attrs[var]:\n", " mattrval = merged_attrs[var][attr] + attrval\n", " mattridx = mattrs_idx[var][attr] + attridx\n", " \n", " _,idx = np.unique(mattridx, return_index=True)\n", " attridx = [mattridx[i] for i in idx]\n", " attrval = [mattrval[i] for i in idx]\n", " mattrs_idx = assoc_in(\n", " mattrs_idx, [var,attr], attridx\n", " )\n", " merged_attrs = assoc_in(\n", " merged_attrs, [var,attr], attrval\n", " )\n", " return dslist, merged_attrs\n", " " ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:22.556673Z", "start_time": "2024-08-07T08:28:22.396563Z" }, "tags": [ "hide-output" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'site': [np.str_('testsite1'), np.str_('testsite2')]}\n", "{'ghi': {'serial': [np.str_('S12128.001'), np.str_('test_ghi_ds3')]}, 'gti': {'serial': [np.str_('S12137.049'), np.str_('test_gti_ds3')]}, 'gti_min': {'serial': [np.str_('S12137.049'), np.str_('test_gti_ds3')]}, 'ghi_min': {'serial': [np.str_('S12128.001'), np.str_('test_ghi_ds3')]}, 'gti_max': {'serial': [np.str_('S12137.049'), np.str_('test_gti_ds3')]}, 'ghi_max': {'serial': [np.str_('S12128.001'), np.str_('test_ghi_ds3')]}, 'gti_std': {'serial': [np.str_('S12137.049'), np.str_('test_gti_ds3')]}, 'ghi_std': {'serial': [np.str_('S12128.001'), np.str_('test_ghi_ds3')]}, 'maintenance_flag_ghi': {'note_general': [np.str_('222'), np.str_('222')]}, 'maintenance_flag_gti': {'note_general': [np.str_('222'), np.str_('222')]}}\n" ] } ], "source": [ "#|dropout\n", "merged_gattrs = _merge_gattrs_by_station(dslist, merge_gattrs={\"site\":\"\"})\n", "print(merged_gattrs)\n", "_, merged_attrs = _merge_vattrs_by_station(\n", " dslist,\n", " merge_attrs={\n", " \"serial\":dict(\n", " apply_to=[\"ghi\",\"gti\"],\n", " fill_value=\"\"\n", " ),\n", " \"note_general\":dict(\n", " apply_to=[\"maintenance\"],\n", " fill_value=\"\"\n", " ),})\n", "print(merged_attrs)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Dataset unification\n", "Reindex all kinds of dimension to unify datasets." ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:22.691471Z", "start_time": "2024-08-07T08:28:22.570336Z" }, "tags": [ "hide-input" ] }, "outputs": [], "source": [ "#|export\n", "#|dropcode\n", "def _reindex_time(dslist, freq='1s', timevar='time'):\n", " dates = []\n", " for i in range(len(dslist)):\n", " # reindex to full day\n", " dates.append(dslist[i][timevar].values[0].astype(\"datetime64[D]\"))\n", " udates = np.unique(dates)\n", " \n", " timeidx = pd.DatetimeIndex([])\n", " for date in udates:\n", " timeidx = timeidx.append(pd.date_range(\n", " date,\n", " date + np.timedelta64(1, 'D'),\n", " freq=freq,\n", " inclusive='left'\n", " ))\n", " \n", " for i in range(len(dslist)):\n", " dslist[i] = dslist[i].reindex(\n", " {timevar:timeidx},\n", " method='nearest',\n", " tolerance=np.timedelta64(1, 'ms')\n", " )\n", " return dslist\n", "\n", "def _reindex_station(dslist):\n", " stations = []\n", " for i in range(len(dslist)):\n", " # reindex to full day\n", " stations += list(dslist[i].station.values)\n", " ustations = np.unique(stations)\n", " for i in range(len(dslist)):\n", " dslist[i] = dslist[i].reindex(\n", " {\"station\":ustations},\n", " method='nearest',\n", " tolerance=1e-6\n", " )\n", " return dslist\n", "\n", "def _reindex_maintenancetime(dslist):\n", " mtimes = []\n", " for i in range(len(dslist)):\n", " # reindex to full day\n", " mtimes += list(dslist[i].maintenancetime.values)\n", " umtimes = np.unique(mtimes)\n", " for i in range(len(dslist)):\n", " dslist[i] = dslist[i].reindex(\n", " {\"maintenancetime\":umtimes},\n", " method='nearest',\n", " tolerance=np.timedelta64(1, 'ms')\n", " )\n", " return dslist" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Snap Maintenance time coordinate\n", "The Maintenancetime coordinate is given at report time, which is actually somewhere after the maintenance. Assuming the report takes place on same or next day, we snap the coordinate to matching data gaps of 10-120min. " ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:22.910708Z", "start_time": "2024-08-07T08:28:22.771500Z" }, "tags": [ "hide-input" ] }, "outputs": [], "source": [ "#|export\n", "#|dropcode\n", "def _maintenancetime_snap_to_gap(ds):\n", " old_mtimes = ds.maintenancetime.values\n", " new_mtimes = old_mtimes.copy()\n", " for i, mtime in enumerate(old_mtimes):\n", " dtime = mtime.astype(\"datetime64[D]\") - ds.time.values[0].astype(\"datetime64[D]\")\n", " # consider only same or next day maintenance times for snapping\n", " if (dtime > np.timedelta64(1,\"D\") or\n", " dtime < np.timedelta64(0,\"D\")):\n", " continue\n", " # lookup station\n", " istation = np.argwhere(~np.isnan(\n", " ds.maintenance_flag_ghi.values[i,:]\n", " ))[0][0]\n", " # identify gaps and gap length\n", " igap = np.argwhere(np.isnan(\n", " ds.ghi.isel(station=istation).values\n", " )).ravel()\n", " if len(igap)<10:\n", " continue\n", " digap = np.diff(igap)\n", " istartgaps = np.insert(digap,0,0)!=1\n", " iendgaps = np.insert(digap,-1,0)!=1\n", " # checkout only gaps of certain length\n", " gaptimes = []\n", " gapidxs = []\n", " for istart,iend in zip(igap[istartgaps],igap[iendgaps]):\n", " gaptime = ds.time.values[iend] - ds.time.values[istart]\n", " # assume maintenance takes at least 10 min and maximum 2h\n", " if (np.timedelta64(10, \"m\") < gaptime < np.timedelta64(2, 'h')):\n", " gapidxs.append(istart)\n", " gaptimes.append(ds.time.values[istart])\n", " if len(gaptimes)==0:\n", " continue\n", " # snap maintenance time to the closest matching gap\n", " new_mtimes[i] = gaptimes[np.argmin([mtime-gtime for gtime in gaptimes])]\n", " # update maintenancetime coordinate\n", " ds = ds.assign_coords({\n", " \"maintenancetime\": (\"maintenancetime\", new_mtimes)\n", " })\n", " ds = ds.sortby(\"maintenancetime\")\n", " \n", " # merge duplicates\n", " dates, counts = np.unique(ds.maintenancetime.values,return_counts=True)\n", " duplets = dates[counts>1]\n", " if len(duplets)>0:\n", " dsold = ds.copy()\n", " ds = ds.drop_duplicates(dim='maintenancetime', keep='first')\n", " for ddate in duplets:\n", " dst = dsold.sel(maintenancetime=ddate)\n", " vars = [var for var in dst if \"maintenancetime\" in dst[var].dims]\n", " for var in vars:\n", " for i in range(dst.maintenancetime.size-1):\n", " mask = np.isnan(ds[var].sel(maintenancetime=ddate).values)\n", " ds[var].sel(maintenancetime=ddate).values[mask] = dst[var].isel(maintenancetime=i+1).values[mask]\n", " \n", " return ds" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Merging processed datasets" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:23.177212Z", "start_time": "2024-08-07T08:28:22.936803Z" }, "tags": [ "hide-input" ] }, "outputs": [], "source": [ "#|export\n", "#|dropcode\n", "def merge_l1b(\n", " dslist,\n", " freq='1s',\n", " timevar='time',\n", " merge_gattrs={\"site\":\"\"},\n", " merge_attrs={\n", " \"calibration_Cabsolute\":dict(\n", " fill_value=0,\n", " apply_to=[\"ghi\", \"gti\"]\n", " ),\n", " \"serial\":dict(\n", " fill_value=\"\",\n", " apply_to=[\"ghi\", \"gti\"]\n", " ),\n", " \"vangle\":dict(\n", " fill_value=0,\n", " apply_to=[\"ghi\", \"gti\"]\n", " ),\n", " \"hangle\":dict(\n", " fill_value=0,\n", " apply_to=[\"ghi\", \"gti\"]\n", " ),\n", " \"note_general\":dict(\n", " fill_value=\"\",\n", " apply_to=[\"maintenance\"]\n", " ),\n", " \"note_clean\":dict(\n", " fill_value=\"\",\n", " apply_to=[\"maintenance\"]\n", " ),\n", " \"note_level\":dict(\n", " fill_value=\"\",\n", " apply_to=[\"maintenance\"]\n", " ),\n", " }\n", "):\n", " logger.info(f\"Merging {len(dslist)} datasets.\")\n", " # sort by first station coordinate\n", " dslist = _sort_by_station(dslist)\n", " \n", " ################################################################\n", " ## Merge Attributes\n", " # have to merge attributes before reindexing, \n", " # as attributes are tied the specific stations in the current datasets\n", " merged_gattrs = _merge_gattrs_by_station(dslist, merge_gattrs=merge_gattrs)\n", " dslist, merged_attrs = _merge_vattrs_by_station(dslist, merge_attrs=merge_attrs)\n", " \n", " #####################################################################\n", " ## Unify datasets\n", " # reindex timevar:\n", " dslist = _reindex_time(dslist, freq=freq, timevar=timevar)\n", " # reindex station var:\n", " dslist = _reindex_station(dslist)\n", " # reindex maintenancetime var:\n", " dslist = _reindex_maintenancetime(dslist)\n", " \n", " #####################################################################\n", " ## Merge datasets\n", " # merge vars with (time,station) dims\n", " for i in range(len(dslist)):\n", " dst = dslist[i].copy()\n", " dst = dst.drop_vars(\n", " [var for var in dst if not (timevar in dst[var].dims and \"station\" in dst[var].dims)]\n", " )\n", " if i==0:\n", " ds_time_station = dst.copy()\n", " else:\n", " # handle overlapping values by dropping from the first (override from second)\n", " for var in dst:\n", " overlap = (~np.isnan(dst[var].values))*(~np.isnan(dst[var].values))\n", " ds_time_station[var].values = ds_time_station[var].values.astype(float)\n", " ds_time_station[var].values[overlap] = np.nan\n", " ds_time_station = ds_time_station.merge(dst,compat='no_conflicts')\n", " \n", " # merge vars with (maintenancetime, station) dims\n", " for i in range(len(dslist)):\n", " dst = dslist[i].copy()\n", " dst = dst.drop_vars(\n", " [var for var in dst if not (\"maintenancetime\" in dst[var].dims and \"station\" in dst[var].dims)]\n", " )\n", " if i==0:\n", " ds_mtime_station = dst.copy()\n", " else:\n", " for var in dst:\n", " overlap = (~np.isnan(dst[var].values))*(~np.isnan(dst[var].values))\n", " ds_mtime_station[var].values = ds_mtime_station[var].values.astype(float)\n", " ds_mtime_station[var].values[overlap] = np.nan\n", " ds_mtime_station = ds_mtime_station.merge(dst,compat='no_conflicts')\n", " \n", " # merge vars with (station) dims\n", " for i in range(len(dslist)):\n", " dst = dslist[i].copy()\n", " dst = dst.drop_vars(\n", " [var for var in dst if not (len(dst[var].dims)==1 and \"station\" in dst[var].dims)]\n", " )\n", " if i==0:\n", " ds_station = dst.copy()\n", " else:\n", " try:\n", " # works there is no overlap with non null values ( new stations )\n", " ds_station = ds_station.merge(dst, compat='no_conflicts')\n", " except:\n", " # override if station already exists\n", " # but fill nan values if available in second dataset\n", " ustations = np.unique(list(ds_station.station.values)+list(dst.station.values))\n", " ds_station = ds_station.reindex(station=ustations)\n", " dst = dst.reindex(station=ustations)\n", " for key in list(set(ds_station.keys())&set(dst.keys())):\n", " mask = np.isnan(ds_station[key].values)\n", " ds_station[key].values[mask] = dst[key].values[mask]\n", " ds_station = ds_station.merge(dst, compat='override')\n", " \n", " ds_merged = xr.merge([ds_time_station,ds_station,ds_mtime_station],compat='no_conflicts')\n", " \n", " ###########################################################################\n", " ## add merged attrs\n", " # write new history\n", " now = pd.to_datetime(np.datetime64(\"now\"))\n", " ds_merged.attrs[\"history\"] = f\"{now.isoformat()}: Merged level l1b by pyrnet version {pyrnet_version}; \"\n", " \n", " # save merged global attrs\n", " for attr in ds_merged.attrs:\n", " if attr not in merge_gattrs:\n", " continue\n", " ds_merged.attrs[attr] = merged_gattrs[attr]\n", " \n", " # save merged variable attrs\n", " for var in ds_merged:\n", " for attr in ds_merged[var].attrs:\n", " if attr not in merge_attrs:\n", " continue\n", " ds_merged[var].attrs[attr] = merged_attrs[var][attr]\n", " \n", " #############################################################################\n", " ## maintenance time snap to data gap\n", " # check maintenance time per station\n", " # if its in time range, scan for gaps >10min and snap closest maintenance time to this gaps\n", " ds_merged = _maintenancetime_snap_to_gap(ds_merged)\n", " \n", " # update automatic quality flags\n", " ds_merged = pyrnet.qcrad.add_qc_flags(ds_merged, [\"ghi\",\"gti\"])\n", " # add encoding\n", " ds_merged = add_encoding(ds_merged)\n", " logger.info(\"... merging done.\")\n", " return ds_merged\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test merging" ] }, { "cell_type": "code", "execution_count": 50, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:33.437250Z", "start_time": "2024-08-07T08:28:23.189922Z" }, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:75: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_time_station = ds_time_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:90: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_mtime_station = ds_mtime_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:90: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_mtime_station = ds_mtime_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:90: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_mtime_station = ds_mtime_station.merge(dst)\n", "/var/folders/p8/dnvld1zx4cz_ykgl57m9dz2m0000gp/T/ipykernel_11929/2319986636.py:90: FutureWarning: In a future version of xarray the default value for compat will change from compat='no_conflicts' to compat='override'. This is likely to lead to different results when combining overlapping variables with the same name. To opt in to new defaults and get rid of these warnings now use `set_options(use_new_combine_kwarg_defaults=True) or set compat explicitly.\n", " ds_mtime_station = ds_mtime_station.merge(dst)\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.Dataset> Size: 1kB\n",
       "Dimensions:               (station: 1, time: 9, maintenancetime: 1)\n",
       "Coordinates:\n",
       "  * station               (station) float32 4B 1.0\n",
       "  * time                  (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n",
       "  * maintenancetime       (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
       "Data variables: (12/20)\n",
       "    ghi                   (time, station) float64 72B 280.9 280.9 ... 280.9\n",
       "    gti                   (time, station) float64 72B 289.0 289.0 ... 289.0\n",
       "    ta                    (time, station) float64 72B 294.8 294.7 ... 294.8\n",
       "    rh                    (time, station) float64 72B 0.6253 0.6248 ... 0.6253\n",
       "    battery_voltage       (time, station) float64 72B 6.443 6.445 ... 6.443\n",
       "    gti_min               (time, station) float64 72B 288.6 288.6 ... 288.6\n",
       "    ...                    ...\n",
       "    qc_flag_gti           (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
       "    lat                   (station) float64 8B 51.39\n",
       "    lon                   (station) float64 8B 11.89\n",
       "    esd                   (station) float64 8B 1.01\n",
       "    maintenance_flag_ghi  (maintenancetime, station) float64 8B 9.0\n",
       "    maintenance_flag_gti  (maintenancetime, station) float64 8B 7.0\n",
       "Attributes: (12/32)\n",
       "    Conventions:               CF-1.10, ACDD-1.3\n",
       "    title:                     TROPOS pyranometer network (PyrNet) observatio...\n",
       "    history:                   2025-10-16T11:47:26: Merged level l1b by pyrne...\n",
       "    institution:               Leibniz Institute for Tropospheric Research (T...\n",
       "    source:                    TROPOS pyranometer network (PyrNet)\n",
       "    references:                https://doi.org/10.5194/amt-9-1153-2016\n",
       "    ...                        ...\n",
       "    geospatial_lon_units:      degE\n",
       "    time_coverage_start:       2022-08-30T11:21:01\n",
       "    time_coverage_end:         2022-08-30T11:21:09\n",
       "    time_coverage_duration:    P0DT0H0M8S\n",
       "    time_coverage_resolution:  P0DT0H0M1S\n",
       "    site:                      [np.str_('testsite1'), np.str_('testsite2')]
" ], "text/plain": [ " Size: 1kB\n", "Dimensions: (station: 1, time: 9, maintenancetime: 1)\n", "Coordinates:\n", " * station (station) float32 4B 1.0\n", " * time (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n", " * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n", "Data variables: (12/20)\n", " ghi (time, station) float64 72B 280.9 280.9 ... 280.9\n", " gti (time, station) float64 72B 289.0 289.0 ... 289.0\n", " ta (time, station) float64 72B 294.8 294.7 ... 294.8\n", " rh (time, station) float64 72B 0.6253 0.6248 ... 0.6253\n", " battery_voltage (time, station) float64 72B 6.443 6.445 ... 6.443\n", " gti_min (time, station) float64 72B 288.6 288.6 ... 288.6\n", " ... ...\n", " qc_flag_gti (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n", " lat (station) float64 8B 51.39\n", " lon (station) float64 8B 11.89\n", " esd (station) float64 8B 1.01\n", " maintenance_flag_ghi (maintenancetime, station) float64 8B 9.0\n", " maintenance_flag_gti (maintenancetime, station) float64 8B 7.0\n", "Attributes: (12/32)\n", " Conventions: CF-1.10, ACDD-1.3\n", " title: TROPOS pyranometer network (PyrNet) observatio...\n", " history: 2025-10-16T11:47:26: Merged level l1b by pyrne...\n", " institution: Leibniz Institute for Tropospheric Research (T...\n", " source: TROPOS pyranometer network (PyrNet)\n", " references: https://doi.org/10.5194/amt-9-1153-2016\n", " ... ...\n", " geospatial_lon_units: degE\n", " time_coverage_start: 2022-08-30T11:21:01\n", " time_coverage_end: 2022-08-30T11:21:09\n", " time_coverage_duration: P0DT0H0M8S\n", " time_coverage_resolution: P0DT0H0M1S\n", " site: [np.str_('testsite1'), np.str_('testsite2')]" ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#|dropout\n", "#|dropcode\n", "ds_merged = merge_l1b(dslist,freq='1s',timevar='time')\n", "ds_merged.reindex_like(dslist[0])" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:28:33.528443Z", "start_time": "2024-08-07T08:28:33.466118Z" }, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "# #|export\n", "# def merge_ds(ds1, ds2, timevar=\"time\"):\n", "# \"\"\"Merge two datasets along the time dimension.\n", "# \"\"\"\n", "# if ds1[timevar].equals(ds2[timevar]):\n", "# logging.info(\"Overwrite existing file.\")\n", "# return ds2\n", "# logging.info(\"Merge with existing file.\")\n", "# \n", "# ## overwrite non time dependent variables\n", "# overwrite_vars = [ v for v in ds1 if timevar not in ds1[v].dims ]\n", "# \n", "# ## merge both datasets\n", "# ds_new=ds1.merge(ds2,\n", "# compat='no_conflicts',\n", "# overwrite_vars=overwrite_vars)\n", "# \n", "# # add global coverage attributes\n", "# ds_new.attrs.update({'merged':1})\n", "# \n", "# # add encoding again\n", "# ds_new = add_encoding(ds_new)\n", "# return ds_new" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "ExecuteTime": { "end_time": "2024-08-07T08:29:09.766249Z", "start_time": "2024-08-07T08:28:33.597863Z" }, "tags": [ "remove-cell", "hide-input", "hide-output" ] }, "outputs": [], "source": [ "#|hide\n", "# Export module\n", "# Requires *nbdev* to export and update the *../lib/logger.py* module\n", "import nbdev.export\n", "import nbformat as nbf\n", "name = \"data\"\n", "\n", "# Export python module\n", "nbdev.export.nb_export( f\"{name}.ipynb\" ,f\"../../src/pyrnet\")\n", "\n", "# Export to docs\n", "ntbk = nbf.read(f\"{name}.ipynb\", nbf.NO_CONVERT)\n", "\n", "text_search_dict = {\n", " \"#|hide\": \"remove-cell\", # Remove the whole cell\n", " \"#|dropcode\": \"hide-input\", # Hide the input w/ a button to show\n", " \"#|dropout\": \"hide-output\" # Hide the output w/ a button to show\n", "}\n", "for cell in ntbk.cells:\n", " cell_tags = cell.get('metadata', {}).get('tags', [])\n", " for key, val in text_search_dict.items():\n", " if key in cell['source']:\n", " if val not in cell_tags:\n", " cell_tags.append(val)\n", " if len(cell_tags) > 0:\n", " cell['metadata']['tags'] = cell_tags\n", " nbf.write(ntbk, f\"../../docs/source/nbs/{name}.ipynb\")" ] } ], "metadata": { "kernelspec": { "display_name": "pyrnet", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.7" } }, "nbformat": 4, "nbformat_minor": 4 }