{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "419674db-073f-4f76-9770-e3ca06e6db60",
   "metadata": {},
   "source": [
    "# Jupyter Notebook - Retrieve Data\n",
    "\n",
    "## SDS & Archiver Data Retrieval\n",
    "Get data from two sources, specify a time range."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6a8fde7-05c3-4690-a871-4ee4d107d66c",
   "metadata": {},
   "source": [
    "### Import modules\n",
    "Need a data reading function from ops_data_client:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4cc12f93-f7cf-4547-a9f1-9653e5492e49",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ops_data_client import get_data\n",
    "import polars as pl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "627a6108-d63b-446e-b8de-e3ac6e10d442",
   "metadata": {},
   "source": [
    "### Get PV data\n",
    "Using the full PV names, get data from the SDS, which is returned as a Polars Dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ec57cb1b-7191-4995-8ebf-36a62b937ad5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = get_data(\n",
    "    pvs=[\"RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData\",\n",
    "         \"PBI-APTM02:Ctrl-ECAT-100:WF-Temp11\"],\n",
    "    start=\"2025-06-11 04:06:25.925641123\",\n",
    "    end=\"2025-06-11 04:06:26.782784123456\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a3717bd-24ef-41e0-8d16-32f605c30446",
   "metadata": {},
   "source": [
    "### Display data\n",
    "Simply print the dataframe in its default table format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aad708d5-7801-44e5-b68a-555cfa6dfa78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (16, 36)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>PBI-APTM02:Ctrl-ECAT-100:WF-Temp11_data_array</th><th>PBI-APTM02:Ctrl-ECAT-100:WF-Temp11_severity</th><th>PBI-APTM02:Ctrl-ECAT-100:WF-Temp11_status</th><th>PBI-APTM02:Ctrl-ECAT-100:WF-Temp11_field_values</th><th>PBI-APTM02:Ctrl-ECAT-100:WF-Temp11_data_ts</th><th>cycle_id</th><th>cycle_ts</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_data_ts</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_stored_at</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_collector</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_ctype</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_pv</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_data_array</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_event_id</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_event_count</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_event_code</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_event_ts</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_sds_cycle_id</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_sds_cycle_ts</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_beam_mode</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_beam_state</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_beam_present</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_beam_len</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_beam_energy</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_beam_current</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_beam_dest</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_buffer_size</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_buffer_index</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_array_sampling</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_array_decimation</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_array_size</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_trig_name</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_trig_evr</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_trig_delay</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_trig_code</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_trig_ts</th></tr><tr><td>list[f64]</td><td>i64</td><td>i64</td><td>list[null]</td><td>datetime[ns, UTC]</td><td>i64</td><td>datetime[ns, UTC]</td><td>datetime[ns, UTC]</td><td>datetime[μs, UTC]</td><td>str</td><td>str</td><td>str</td><td>list[f32]</td><td>u64</td><td>u64</td><td>u64</td><td>datetime[ns, UTC]</td><td>u64</td><td>datetime[ns, UTC]</td><td>str</td><td>str</td><td>bool</td><td>u32</td><td>u32</td><td>u32</td><td>str</td><td>u32</td><td>u32</td><td>f32</td><td>u32</td><td>u32</td><td>str</td><td>str</td><td>f32</td><td>u64</td><td>datetime[ns, UTC]</td></tr></thead><tbody><tr><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>15658604172</td><td>2025-06-11 04:06:25.854212373 UTC</td><td>2025-06-11 04:06:25.854212352 UTC</td><td>2025-06-11 04:10:03 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[4.908, 4.898, … 4.918]</td><td>15658604183</td><td>18819282</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604172</td><td>2025-06-11 04:06:25.854212352 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>0</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:25.925640960 UTC</td></tr><tr><td>[21.791, 23.152, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:25.925640941 UTC</td><td>15658604173</td><td>2025-06-11 04:06:25.925640941 UTC</td><td>2025-06-11 04:06:25.925640960 UTC</td><td>2025-06-11 04:10:03 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[4.898, 4.918, … 4.778]</td><td>15658604183</td><td>18819284</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604173</td><td>2025-06-11 04:06:25.925640960 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>1</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:25.997069568 UTC</td></tr><tr><td>[20.950001, 23.01, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:25.997069509 UTC</td><td>15658604174</td><td>2025-06-11 04:06:25.997069509 UTC</td><td>2025-06-11 04:06:25.997069568 UTC</td><td>2025-06-11 04:10:03 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[5.028, 4.998, … 4.988]</td><td>15658604183</td><td>18819284</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604174</td><td>2025-06-11 04:06:25.997069568 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>2</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:26.068498176 UTC</td></tr><tr><td>[20.929001, 19.315001, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:26.068498077 UTC</td><td>15658604175</td><td>2025-06-11 04:06:26.068498077 UTC</td><td>2025-06-11 04:06:26.068498176 UTC</td><td>2025-06-11 04:10:03 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[4.898, 4.888, … 4.888]</td><td>15658604183</td><td>18819284</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604175</td><td>2025-06-11 04:06:26.068498176 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>3</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:26.139926528 UTC</td></tr><tr><td>[18.632, 20.610001, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:26.139926646 UTC</td><td>15658604176</td><td>2025-06-11 04:06:26.139926646 UTC</td><td>2025-06-11 04:06:26.139926528 UTC</td><td>2025-06-11 04:10:03 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[4.918, 4.828, … 4.878]</td><td>15658604183</td><td>18819286</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604176</td><td>2025-06-11 04:06:26.139926528 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>4</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:26.211355136 UTC</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>[18.878, 22.485001, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:26.639926623 UTC</td><td>15658604183</td><td>2025-06-11 04:06:26.639926623 UTC</td><td>2025-06-11 04:06:26.639926528 UTC</td><td>2025-06-11 04:10:03 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[5.018, 4.998, … 4.828]</td><td>15658604183</td><td>18819292</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604183</td><td>2025-06-11 04:06:26.639926528 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>11</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:26.711355136 UTC</td></tr><tr><td>[18.878, 22.485001, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:26.639926623 UTC</td><td>15658604183</td><td>2025-06-11 04:06:26.639926623 UTC</td><td>2025-06-11 04:06:26.639926528 UTC</td><td>2025-06-11 04:10:03 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[4.898, 4.858, … 4.958]</td><td>15658604183</td><td>18819292</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604183</td><td>2025-06-11 04:06:26.639926528 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>12</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:26.782783744 UTC</td></tr><tr><td>[22.555, 22.573999, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:26.711355191 UTC</td><td>15658604184</td><td>2025-06-11 04:06:26.711355191 UTC</td><td>2025-06-11 04:06:26.711355136 UTC</td><td>2025-06-11 04:10:03 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[5.168, 5.148, … 4.928]</td><td>15658604183</td><td>18819294</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604184</td><td>2025-06-11 04:06:26.711355136 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>13</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:26.782783744 UTC</td></tr><tr><td>[22.555, 22.573999, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:26.711355191 UTC</td><td>15658604184</td><td>2025-06-11 04:06:26.711355191 UTC</td><td>2025-06-11 04:06:26.711355136 UTC</td><td>2025-06-11 09:33:09 UTC</td><td>&quot;RFQ-010-FIM-DOD&quot;</td><td>&quot;DD&quot;</td><td>&quot;RFQ-010:RFS-Cav-110:Fld-Wave_S…</td><td>[5.168, 5.148, … 4.928]</td><td>15658604183</td><td>18819294</td><td>42</td><td>2025-06-11 04:06:26.668351744 UTC</td><td>15658604184</td><td>2025-06-11 04:06:26.711355136 UTC</td><td>&quot;Probe&quot;</td><td>&quot;BEAM_PRESENT_YES_FAKE&quot;</td><td>false</td><td>0</td><td>0</td><td>59300</td><td>&quot;LebtFc&quot;</td><td>14</td><td>13</td><td>1e6</td><td>1</td><td>8000</td><td>&quot;RFQ-010:RFS-EVR-101&quot;</td><td>&quot;DlyGen-6&quot;</td><td>-3000.0</td><td>14</td><td>2025-06-11 04:06:26.782783744 UTC</td></tr><tr><td>[20.827, 21.809, … 0.0]</td><td>0</td><td>0</td><td>[]</td><td>2025-06-11 04:06:26.782783759 UTC</td><td>15658604185</td><td>2025-06-11 04:06:26.782783759 UTC</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (16, 36)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ PBI-APTM0 ┆ PBI-APTM0 ┆ PBI-APTM0 ┆ PBI-APTM0 ┆ … ┆ RFQ-010:R ┆ RFQ-010:R ┆ RFQ-010:R ┆ RFQ-010: │\n",
       "│ 2:Ctrl-EC ┆ 2:Ctrl-EC ┆ 2:Ctrl-EC ┆ 2:Ctrl-EC ┆   ┆ FS-Cav-11 ┆ FS-Cav-11 ┆ FS-Cav-11 ┆ RFS-Cav- │\n",
       "│ AT-100:WF ┆ AT-100:WF ┆ AT-100:WF ┆ AT-100:WF ┆   ┆ 0:Fld-Wav ┆ 0:Fld-Wav ┆ 0:Fld-Wav ┆ 110:Fld- │\n",
       "│ -Te…      ┆ -Te…      ┆ -Te…      ┆ -Te…      ┆   ┆ e_S…      ┆ e_S…      ┆ e_S…      ┆ Wave_S…  │\n",
       "│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---      │\n",
       "│ list[f64] ┆ i64       ┆ i64       ┆ list[null ┆   ┆ str       ┆ f32       ┆ u64       ┆ datetime │\n",
       "│           ┆           ┆           ┆ ]         ┆   ┆           ┆           ┆           ┆ [ns,     │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ UTC]     │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ null      ┆ null      ┆ null      ┆ null      ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :25.9256 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 40960 …  │\n",
       "│ [21.791,  ┆ 0         ┆ 0         ┆ []        ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│ 23.152, … ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│ 0.0]      ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :25.9970 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 69568 …  │\n",
       "│ [20.95000 ┆ 0         ┆ 0         ┆ []        ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│ 1, 23.01, ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│ … 0.0]    ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :26.0684 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 98176 …  │\n",
       "│ [20.92900 ┆ 0         ┆ 0         ┆ []        ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│ 1, 19.315 ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│ 001, …    ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :26.1399 │\n",
       "│ 0.0]      ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 26528 …  │\n",
       "│ [18.632,  ┆ 0         ┆ 0         ┆ []        ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│ 20.610001 ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│ , … 0.0]  ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :26.2113 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 55136 …  │\n",
       "│ …         ┆ …         ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …        │\n",
       "│ [18.878,  ┆ 0         ┆ 0         ┆ []        ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│ 22.485001 ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│ , … 0.0]  ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :26.7113 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 55136 …  │\n",
       "│ [18.878,  ┆ 0         ┆ 0         ┆ []        ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│ 22.485001 ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│ , … 0.0]  ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :26.7827 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 83744 …  │\n",
       "│ [22.555,  ┆ 0         ┆ 0         ┆ []        ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│ 22.573999 ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│ , … 0.0]  ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :26.7827 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 83744 …  │\n",
       "│ [22.555,  ┆ 0         ┆ 0         ┆ []        ┆ … ┆ DlyGen-6  ┆ -3000.0   ┆ 14        ┆ 2025-06- │\n",
       "│ 22.573999 ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 11 04:06 │\n",
       "│ , … 0.0]  ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ :26.7827 │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 83744 …  │\n",
       "│ [20.827,  ┆ 0         ┆ 0         ┆ []        ┆ … ┆ null      ┆ null      ┆ null      ┆ null     │\n",
       "│ 21.809, … ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ 0.0]      ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58d9ad1c-c7d3-4eaf-b032-c6457847bad7",
   "metadata": {},
   "source": [
    "### Simple calculations\n",
    "Find the mean values in the data array for each PV in each cycle, then display just some of the dataframe columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b8659588-2be9-4d90-85d6-b74d9e430044",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (16, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>cycle_id</th><th>cycle_ts</th><th>RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_data_array</th><th>PBI-APTM02:Ctrl-ECAT-100:WF-Temp11_data_array</th><th>sds_mean_value</th><th>arch_mean_value</th></tr><tr><td>i64</td><td>datetime[ns, UTC]</td><td>list[f32]</td><td>list[f64]</td><td>f32</td><td>f64</td></tr></thead><tbody><tr><td>15658604172</td><td>2025-06-11 04:06:25.854212373 UTC</td><td>[4.908, 4.898, … 4.918]</td><td>null</td><td>4.888259</td><td>null</td></tr><tr><td>15658604173</td><td>2025-06-11 04:06:25.925640941 UTC</td><td>[4.898, 4.918, … 4.778]</td><td>[21.791, 23.152, … 0.0]</td><td>4.854619</td><td>6.180347</td></tr><tr><td>15658604174</td><td>2025-06-11 04:06:25.997069509 UTC</td><td>[5.028, 4.998, … 4.988]</td><td>[20.950001, 23.01, … 0.0]</td><td>4.984537</td><td>6.302639</td></tr><tr><td>15658604175</td><td>2025-06-11 04:06:26.068498077 UTC</td><td>[4.898, 4.888, … 4.888]</td><td>[20.929001, 19.315001, … 0.0]</td><td>4.881917</td><td>6.740958</td></tr><tr><td>15658604176</td><td>2025-06-11 04:06:26.139926646 UTC</td><td>[4.918, 4.828, … 4.878]</td><td>[18.632, 20.610001, … 0.0]</td><td>4.893318</td><td>6.561056</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>15658604183</td><td>2025-06-11 04:06:26.639926623 UTC</td><td>[5.018, 4.998, … 4.828]</td><td>[18.878, 22.485001, … 0.0]</td><td>4.925144</td><td>6.506083</td></tr><tr><td>15658604183</td><td>2025-06-11 04:06:26.639926623 UTC</td><td>[4.898, 4.858, … 4.958]</td><td>[18.878, 22.485001, … 0.0]</td><td>4.911372</td><td>6.506083</td></tr><tr><td>15658604184</td><td>2025-06-11 04:06:26.711355191 UTC</td><td>[5.168, 5.148, … 4.928]</td><td>[22.555, 22.573999, … 0.0]</td><td>4.955574</td><td>6.503056</td></tr><tr><td>15658604184</td><td>2025-06-11 04:06:26.711355191 UTC</td><td>[5.168, 5.148, … 4.928]</td><td>[22.555, 22.573999, … 0.0]</td><td>4.955574</td><td>6.503056</td></tr><tr><td>15658604185</td><td>2025-06-11 04:06:26.782783759 UTC</td><td>null</td><td>[20.827, 21.809, … 0.0]</td><td>null</td><td>6.178431</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (16, 6)\n",
       "┌─────────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┐\n",
       "│ cycle_id    ┆ cycle_ts       ┆ RFQ-010:RFS-Ca ┆ PBI-APTM02:Ctr ┆ sds_mean_value ┆ arch_mean_valu │\n",
       "│ ---         ┆ ---            ┆ v-110:Fld-Wave ┆ l-ECAT-100:WF- ┆ ---            ┆ e              │\n",
       "│ i64         ┆ datetime[ns,   ┆ _S…            ┆ Te…            ┆ f32            ┆ ---            │\n",
       "│             ┆ UTC]           ┆ ---            ┆ ---            ┆                ┆ f64            │\n",
       "│             ┆                ┆ list[f32]      ┆ list[f64]      ┆                ┆                │\n",
       "╞═════════════╪════════════════╪════════════════╪════════════════╪════════════════╪════════════════╡\n",
       "│ 15658604172 ┆ 2025-06-11 04: ┆ [4.908, 4.898, ┆ null           ┆ 4.888259       ┆ null           │\n",
       "│             ┆ 06:25.85421237 ┆ … 4.918]       ┆                ┆                ┆                │\n",
       "│             ┆ 3 …            ┆                ┆                ┆                ┆                │\n",
       "│ 15658604173 ┆ 2025-06-11 04: ┆ [4.898, 4.918, ┆ [21.791,       ┆ 4.854619       ┆ 6.180347       │\n",
       "│             ┆ 06:25.92564094 ┆ … 4.778]       ┆ 23.152, … 0.0] ┆                ┆                │\n",
       "│             ┆ 1 …            ┆                ┆                ┆                ┆                │\n",
       "│ 15658604174 ┆ 2025-06-11 04: ┆ [5.028, 4.998, ┆ [20.950001,    ┆ 4.984537       ┆ 6.302639       │\n",
       "│             ┆ 06:25.99706950 ┆ … 4.988]       ┆ 23.01, … 0.0]  ┆                ┆                │\n",
       "│             ┆ 9 …            ┆                ┆                ┆                ┆                │\n",
       "│ 15658604175 ┆ 2025-06-11 04: ┆ [4.898, 4.888, ┆ [20.929001,    ┆ 4.881917       ┆ 6.740958       │\n",
       "│             ┆ 06:26.06849807 ┆ … 4.888]       ┆ 19.315001, …   ┆                ┆                │\n",
       "│             ┆ 7 …            ┆                ┆ 0.0]           ┆                ┆                │\n",
       "│ 15658604176 ┆ 2025-06-11 04: ┆ [4.918, 4.828, ┆ [18.632,       ┆ 4.893318       ┆ 6.561056       │\n",
       "│             ┆ 06:26.13992664 ┆ … 4.878]       ┆ 20.610001, …   ┆                ┆                │\n",
       "│             ┆ 6 …            ┆                ┆ 0.0]           ┆                ┆                │\n",
       "│ …           ┆ …              ┆ …              ┆ …              ┆ …              ┆ …              │\n",
       "│ 15658604183 ┆ 2025-06-11 04: ┆ [5.018, 4.998, ┆ [18.878,       ┆ 4.925144       ┆ 6.506083       │\n",
       "│             ┆ 06:26.63992662 ┆ … 4.828]       ┆ 22.485001, …   ┆                ┆                │\n",
       "│             ┆ 3 …            ┆                ┆ 0.0]           ┆                ┆                │\n",
       "│ 15658604183 ┆ 2025-06-11 04: ┆ [4.898, 4.858, ┆ [18.878,       ┆ 4.911372       ┆ 6.506083       │\n",
       "│             ┆ 06:26.63992662 ┆ … 4.958]       ┆ 22.485001, …   ┆                ┆                │\n",
       "│             ┆ 3 …            ┆                ┆ 0.0]           ┆                ┆                │\n",
       "│ 15658604184 ┆ 2025-06-11 04: ┆ [5.168, 5.148, ┆ [22.555,       ┆ 4.955574       ┆ 6.503056       │\n",
       "│             ┆ 06:26.71135519 ┆ … 4.928]       ┆ 22.573999, …   ┆                ┆                │\n",
       "│             ┆ 1 …            ┆                ┆ 0.0]           ┆                ┆                │\n",
       "│ 15658604184 ┆ 2025-06-11 04: ┆ [5.168, 5.148, ┆ [22.555,       ┆ 4.955574       ┆ 6.503056       │\n",
       "│             ┆ 06:26.71135519 ┆ … 4.928]       ┆ 22.573999, …   ┆                ┆                │\n",
       "│             ┆ 1 …            ┆                ┆ 0.0]           ┆                ┆                │\n",
       "│ 15658604185 ┆ 2025-06-11 04: ┆ null           ┆ [20.827,       ┆ null           ┆ 6.178431       │\n",
       "│             ┆ 06:26.78278375 ┆                ┆ 21.809, … 0.0] ┆                ┆                │\n",
       "│             ┆ 9 …            ┆                ┆                ┆                ┆                │\n",
       "└─────────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_partial = df.with_columns(\n",
    "    (pl.col(\"RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_data_array\").list.mean()).alias(\"sds_mean_value\"),\n",
    "    (pl.col(\"PBI-APTM02:Ctrl-ECAT-100:WF-Temp11_data_array\").list.mean()).alias(\"arch_mean_value\"),\n",
    ").select(\n",
    "      \"cycle_id\", \"cycle_ts\", \"RFQ-010:RFS-Cav-110:Fld-Wave_SDS-DODData_data_array\",\"PBI-APTM02:Ctrl-ECAT-100:WF-Temp11_data_array\", \"sds_mean_value\", \"arch_mean_value\"\n",
    ")\n",
    "\n",
    "df_partial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f908bd0e-da48-4e95-9154-250a11459033",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}