{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "initial_id", "metadata": { "ExecuteTime": { "end_time": "2024-08-06T11:43:07.125944Z", "start_time": "2024-08-06T11:43:07.121495Z" }, "collapsed": true, "tags": [ "remove-cell" ] }, "outputs": [], "source": [ "#|hide\n", "#|default_exp qcrad" ] }, { "cell_type": "markdown", "id": "cf56cd614fb8d48a", "metadata": { "collapsed": false }, "source": [ "# PyrNet automatic quality checks\n", "In the following, functions for automatic quality screening are developed. Including BSRN recommended physical and rare limits, as well as network sanity checks." ] }, { "cell_type": "code", "execution_count": 2, "id": "10e20462eea472cd", "metadata": { "ExecuteTime": { "end_time": "2024-08-06T11:43:10.908256Z", "start_time": "2024-08-06T11:43:07.241659Z" }, "collapsed": false }, "outputs": [], "source": [ "#|export\n", "import xarray as xr\n", "import numpy as np\n", "import warnings\n", "import logging\n", "\n", "import pyrnet.data\n", "import pyrnet.utils\n", "\n", "\n", "# logging setup\n", "logging.basicConfig(\n", " filename='pyrnet.log',\n", " encoding='utf-8',\n", " level=logging.DEBUG,\n", " format='%(asctime)s %(name)s %(levelname)s:%(message)s'\n", ")\n", "logger = logging.getLogger(__name__)" ] }, { "cell_type": "code", "execution_count": 3, "id": "933ea329f4416a5e", "metadata": { "ExecuteTime": { "end_time": "2024-08-06T11:43:11.500362Z", "start_time": "2024-08-06T11:43:10.911578Z" }, "collapsed": false }, "outputs": [], "source": [ "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": 4, "id": "46c99745dd91bc2b", "metadata": { "ExecuteTime": { "end_time": "2024-08-06T11:43:11.505940Z", "start_time": "2024-08-06T11:43:11.502086Z" }, "collapsed": false }, "outputs": [], "source": [ "#|export\n", "class CONSTANTS:\n", " S0 = 1367 # W m-2\n", " k = 5.67*1e-8" ] }, { "cell_type": "markdown", "id": "80c0dbd48c9039c1", "metadata": { "collapsed": false }, "source": [ "## BSRN recommended checks\n", "BSRN recommends thresholds for physical and rare limits of GHI data" ] }, { "cell_type": "code", "execution_count": 5, "id": "f3b0a3fe1b692933", "metadata": { "ExecuteTime": { "end_time": "2024-08-06T11:43:11.515348Z", "start_time": "2024-08-06T11:43:11.509265Z" }, "collapsed": false }, "outputs": [], "source": [ "#|export\n", "class QCCode:\n", " \"\"\" BSRN quality codes\n", " https://wiki.pangaea.de/wiki/BSRN_Toolbox#Quality_Check\n", " \"\"\"\n", " below_physical = 2**0 # 1\n", " above_phyiscal = 2**1 # 2\n", " below_rare = 2**2 # 4\n", " above_rare = 2**3 # 8\n", " compare_to_low = 2**4 # 16\n", " compare_to_high = 2**5 # 32\n", " quality_control_failed = 2**6 # 64" ] }, { "cell_type": "markdown", "id": "94006795", "metadata": {}, "source": [ "## Additional Flags\n", "Outlier and probability of bird or random shading detection" ] }, { "cell_type": "code", "execution_count": 6, "id": "2b9a3dcd", "metadata": {}, "outputs": [], "source": [ "#|export\n", "class FLCode:\n", " \"\"\" Codes for additional pyrnet flags.\n", " \"\"\"\n", " low_outlier = 2**0 # 1\n", " strong_fluctuation = 2**1 # 2 Strong fluctuation, which maybe logger failiure\n", " not_applicable = 2**2" ] }, { "cell_type": "markdown", "id": "66d85261e2c99f10", "metadata": { "collapsed": false }, "source": [ "## Load test dataset:" ] }, { "cell_type": "code", "execution_count": 7, "id": "6dd8e1d8c7a00692", "metadata": { "ExecuteTime": { "end_time": "2024-08-06T11:43:11.693921Z", "start_time": "2024-08-06T11:43:11.517968Z" }, "collapsed": false, "tags": [ "hide-input", "hide-output" ] }, "outputs": [ { "data": { "text/html": [ "
<xarray.Dataset> Size: 1kB\n",
"Dimensions: (station: 1, time: 9, maintenancetime: 1)\n",
"Coordinates:\n",
" * station (station) float64 8B 1.0\n",
" * time (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n",
" * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
"Data variables: (12/20)\n",
" ghi (time, station) float64 72B 280.9 280.9 ... 280.9\n",
" gti (time, station) float64 72B 288.9 288.9 ... 289.4\n",
" ta (time, station) float64 72B 294.8 294.7 ... 294.8\n",
" rh (time, station) float64 72B 0.6253 0.6248 ... 0.629\n",
" battery_voltage (time, station) float64 72B 6.443 6.445 ... 6.465\n",
" gti_min (time, station) float64 72B 288.7 288.7 ... 288.7\n",
" ... ...\n",
" maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n",
" szen (time, station) float64 72B 42.51 42.51 ... 42.51\n",
" sazi (time, station) float64 72B 182.9 182.9 ... 182.9\n",
" esd (station) float64 8B 1.01\n",
" qc_flag_ghi (time, station) float32 36B 0.0 0.0 0.0 ... 0.0 0.0\n",
" qc_flag_gti (time, station) float32 36B 0.0 0.0 0.0 ... 0.0 0.0\n",
"Attributes: (12/31)\n",
" Conventions: CF-1.10, ACDD-1.3\n",
" title: TROPOS pyranometer network (PyrNet) observatio...\n",
" history: 2024-05-31T13:47:06: Generated level l1a by p...\n",
" institution: Leibniz Institute for Tropospheric Research (T...\n",
" source: TROPOS pyranometer network (PyrNet)\n",
" references: https://doi.org/10.5194/amt-9-1153-2016\n",
" ... ...\n",
" geospatial_lon_max: 11.885252\n",
" geospatial_lon_units: degE\n",
" time_coverage_start: 2022-08-30T11:21:01\n",
" time_coverage_end: 2022-08-30T11:21:09\n",
" time_coverage_duration: P0DT0H0M8S\n",
" time_coverage_resolution: P0DT0H0M1S<xarray.Dataset> Size: 1kB\n",
"Dimensions: (station: 1, time: 9, maintenancetime: 1)\n",
"Coordinates:\n",
" * station (station) float64 8B 1.0\n",
" * time (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n",
" * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
"Data variables: (12/22)\n",
" ghi (time, station) float64 72B 280.9 280.9 ... 280.9\n",
" gti (time, station) float64 72B 288.9 288.9 ... 289.4\n",
" ta (time, station) float64 72B 294.8 294.7 ... 294.8\n",
" rh (time, station) float64 72B 0.6253 0.6248 ... 0.629\n",
" battery_voltage (time, station) float64 72B 6.443 6.445 ... 6.465\n",
" gti_min (time, station) float64 72B 288.7 288.7 ... 288.7\n",
" ... ...\n",
" sazi (time, station) float64 72B 182.9 182.9 ... 182.9\n",
" esd (station) float64 8B 1.01\n",
" qc_flag_ghi (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
" qc_flag_gti (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
" add_flag_ghi (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
" add_flag_gti (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
"Attributes: (12/31)\n",
" Conventions: CF-1.10, ACDD-1.3\n",
" title: TROPOS pyranometer network (PyrNet) observatio...\n",
" history: 2024-05-31T13:47:06: Generated level l1a by p...\n",
" institution: Leibniz Institute for Tropospheric Research (T...\n",
" source: TROPOS pyranometer network (PyrNet)\n",
" references: https://doi.org/10.5194/amt-9-1153-2016\n",
" ... ...\n",
" geospatial_lon_max: 11.885252\n",
" geospatial_lon_units: degE\n",
" time_coverage_start: 2022-08-30T11:21:01\n",
" time_coverage_end: 2022-08-30T11:21:09\n",
" time_coverage_duration: P0DT0H0M8S\n",
" time_coverage_resolution: P0DT0H0M1S<xarray.Dataset> Size: 1kB\n",
"Dimensions: (station: 1, time: 9, maintenancetime: 1)\n",
"Coordinates:\n",
" * station (station) float64 8B 1.0\n",
" * time (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n",
" * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
"Data variables: (12/22)\n",
" ghi (time, station) float64 72B 280.9 280.9 ... 280.9\n",
" gti (time, station) float64 72B 288.9 288.9 ... 289.4\n",
" ta (time, station) float64 72B 294.8 294.7 ... 294.8\n",
" rh (time, station) float64 72B 0.6253 0.6248 ... 0.629\n",
" battery_voltage (time, station) float64 72B 6.443 6.445 ... 6.465\n",
" gti_min (time, station) float64 72B 288.7 288.7 ... 288.7\n",
" ... ...\n",
" sazi (time, station) float64 72B 182.9 182.9 ... 182.9\n",
" esd (station) float64 8B 1.01\n",
" qc_flag_ghi (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
" qc_flag_gti (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
" add_flag_ghi (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
" add_flag_gti (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
"Attributes: (12/31)\n",
" Conventions: CF-1.10, ACDD-1.3\n",
" title: TROPOS pyranometer network (PyrNet) observatio...\n",
" history: 2024-05-31T13:47:06: Generated level l1a by p...\n",
" institution: Leibniz Institute for Tropospheric Research (T...\n",
" source: TROPOS pyranometer network (PyrNet)\n",
" references: https://doi.org/10.5194/amt-9-1153-2016\n",
" ... ...\n",
" geospatial_lon_max: 11.885252\n",
" geospatial_lon_units: degE\n",
" time_coverage_start: 2022-08-30T11:21:01\n",
" time_coverage_end: 2022-08-30T11:21:09\n",
" time_coverage_duration: P0DT0H0M8S\n",
" time_coverage_resolution: P0DT0H0M1S<xarray.Dataset> Size: 1kB\n",
"Dimensions: (station: 1, time: 9, maintenancetime: 1)\n",
"Coordinates:\n",
" * station (station) float64 8B 1.0\n",
" * time (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n",
" * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
"Data variables: (12/20)\n",
" ghi (time, station) float64 72B 280.9 280.9 ... 280.9\n",
" gti (time, station) float64 72B 288.9 288.9 ... 289.4\n",
" ta (time, station) float64 72B 294.8 294.7 ... 294.8\n",
" rh (time, station) float64 72B 0.6253 0.6248 ... 0.629\n",
" battery_voltage (time, station) float64 72B 6.443 6.445 ... 6.465\n",
" gti_min (time, station) float64 72B 288.7 288.7 ... 288.7\n",
" ... ...\n",
" maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n",
" szen (time, station) float64 72B 42.51 42.51 ... 42.51\n",
" sazi (time, station) float64 72B 182.9 182.9 ... 182.9\n",
" esd (station) float64 8B 1.01\n",
" qc_flag_ghi (time, station) float32 36B 0.0 0.0 0.0 ... 0.0 0.0\n",
" qc_flag_gti (time, station) float32 36B 0.0 0.0 0.0 ... 0.0 0.0\n",
"Attributes: (12/31)\n",
" Conventions: CF-1.10, ACDD-1.3\n",
" title: TROPOS pyranometer network (PyrNet) observatio...\n",
" history: 2024-05-31T13:47:06: Generated level l1a by p...\n",
" institution: Leibniz Institute for Tropospheric Research (T...\n",
" source: TROPOS pyranometer network (PyrNet)\n",
" references: https://doi.org/10.5194/amt-9-1153-2016\n",
" ... ...\n",
" geospatial_lon_max: 11.885252\n",
" geospatial_lon_units: degE\n",
" time_coverage_start: 2022-08-30T11:21:01\n",
" time_coverage_end: 2022-08-30T11:21:09\n",
" time_coverage_duration: P0DT0H0M8S\n",
" time_coverage_resolution: P0DT0H0M1S<xarray.Dataset> Size: 1kB\n",
"Dimensions: (station: 1, time: 9, maintenancetime: 1)\n",
"Coordinates:\n",
" * station (station) float64 8B 1.0\n",
" * time (time) datetime64[ns] 72B 2022-08-30T11:21:01 ... 2...\n",
" * maintenancetime (maintenancetime) datetime64[ns] 8B 2023-05-08T16:0...\n",
"Data variables: (12/20)\n",
" ghi (time, station) float64 72B 280.9 280.9 ... 280.9\n",
" gti (time, station) float64 72B 288.9 288.9 ... 289.4\n",
" ta (time, station) float64 72B 294.8 294.7 ... 294.8\n",
" rh (time, station) float64 72B 0.6253 0.6248 ... 0.629\n",
" battery_voltage (time, station) float64 72B 6.443 6.445 ... 6.465\n",
" gti_min (time, station) float64 72B 288.7 288.7 ... 288.7\n",
" ... ...\n",
" maintenance_flag_gti (maintenancetime, station) float32 4B 7.0\n",
" szen (time, station) float64 72B 42.51 42.51 ... 42.51\n",
" sazi (time, station) float64 72B 182.9 182.9 ... 182.9\n",
" esd (station) float64 8B 1.01\n",
" qc_flag_ghi (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
" qc_flag_gti (time, station) uint8 9B 0 0 0 0 0 0 0 0 0\n",
"Attributes: (12/31)\n",
" Conventions: CF-1.10, ACDD-1.3\n",
" title: TROPOS pyranometer network (PyrNet) observatio...\n",
" history: 2024-05-31T13:47:06: Generated level l1a by p...\n",
" institution: Leibniz Institute for Tropospheric Research (T...\n",
" source: TROPOS pyranometer network (PyrNet)\n",
" references: https://doi.org/10.5194/amt-9-1153-2016\n",
" ... ...\n",
" geospatial_lon_max: 11.885252\n",
" geospatial_lon_units: degE\n",
" time_coverage_start: 2022-08-30T11:21:01\n",
" time_coverage_end: 2022-08-30T11:21:09\n",
" time_coverage_duration: P0DT0H0M8S\n",
" time_coverage_resolution: P0DT0H0M1S