{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Library Examples" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Archiving information" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "from datetime import datetime, timedelta\n", "\n", "from pytz import UTC\n", "\n", "from epicsarchiver import ArchiverAppliance\n", "\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "archiver = ArchiverAppliance(\"archiver-linac-04.tn.esss.lu.se\")\n", "pv = \"DTL-010:EMR-TT-001:Temp\"" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'lastRotateLogs': 'Never',\n", " 'appliance': 'archiver-linac-02',\n", " 'pvName': 'DTL-010:EMR-TT-001:Temp',\n", " 'pvNameOnly': 'DTL-010:EMR-TT-001:Temp',\n", " 'connectionState': 'true',\n", " 'lastEvent': 'Dec/12/2024 15:32:52 +01:00',\n", " 'samplingPeriod': '0.07',\n", " 'isMonitored': 'true',\n", " 'connectionLastRestablished': 'Sep/12/2024 11:13:11 +02:00',\n", " 'connectionFirstEstablished': 'Aug/27/2024 10:35:52 +02:00',\n", " 'connectionLossRegainCount': '8',\n", " 'status': 'Being archived'}]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "archiver.get_pv_status(pv)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "tags": [ "hide-output" ] }, "outputs": [ { "data": { "text/plain": [ "[{'name': 'PV Name', 'value': 'DTL-010:EMR-TT-001:Temp', 'source': 'mgmt'},\n", " {'name': 'Instance archiving PV',\n", " 'value': 'archiver-linac-02',\n", " 'source': 'mgmt'},\n", " {'name': 'Archival params creation time:',\n", " 'value': 'Mar/09/2023 10:11:56 +01:00',\n", " 'source': 'mgmt'},\n", " {'name': 'Archival params modification time:',\n", " 'value': 'Mar/10/2023 10:00:41 +01:00',\n", " 'source': 'mgmt'},\n", " {'name': 'Archiver DBR type (from typeinfo):',\n", " 'value': 'DBR_SCALAR_DOUBLE',\n", " 'source': 'mgmt'},\n", " {'name': 'Is this a scalar:', 'value': 'Yes', 'source': 'mgmt'},\n", " {'name': 'Number of elements:', 'value': '1', 'source': 'mgmt'},\n", " {'name': 'Precision:', 'value': '0.0', 'source': 'mgmt'},\n", " {'name': 'Units:', 'value': 'degC', 'source': 'mgmt'},\n", " {'name': 'Is this PV paused:', 'value': 'No', 'source': 'mgmt'},\n", " {'name': 'Sampling method:', 'value': 'MONITOR', 'source': 'mgmt'},\n", " {'name': 'Sampling period:', 'value': '0.07', 'source': 'mgmt'},\n", " {'name': 'Are we using PVAccess?', 'value': 'No', 'source': 'mgmt'},\n", " {'name': 'Extra info - MDEL:', 'value': '0.0', 'source': 'mgmt'},\n", " {'name': 'Extra info - ADEL:', 'value': '0.0', 'source': 'mgmt'},\n", " {'name': 'Extra info - SCAN:', 'value': 'Passive', 'source': 'mgmt'},\n", " {'name': 'Extra info - RTYP:', 'value': 'ai', 'source': 'mgmt'},\n", " {'name': 'Archive Fields',\n", " 'value': 'EGU,HIHI,HIGH,ADEL,LOW,LOLO,LOPR,HOPR',\n", " 'source': 'mgmt'},\n", " {'name': 'Channel Name', 'source': 'pv', 'value': 'DTL-010:EMR-TT-001:Temp'},\n", " {'name': 'Host name',\n", " 'source': 'pv',\n", " 'value': 'dtl-cavhl-ipc-ioc.tn.esss.lu.se'},\n", " {'name': 'Controlling PV', 'source': 'pv', 'value': ''},\n", " {'name': 'Is engine currently archiving this?',\n", " 'source': 'pv',\n", " 'value': 'yes'},\n", " {'name': 'Archiver DBR type (initial)',\n", " 'source': 'pv',\n", " 'value': 'DBR_SCALAR_DOUBLE'},\n", " {'name': 'Archiver DBR type (from CA)',\n", " 'source': 'pv',\n", " 'value': 'DBR_SCALAR_DOUBLE'},\n", " {'name': 'Number of elements per event (from CA)',\n", " 'source': 'pv',\n", " 'value': '1'},\n", " {'name': 'Is engine using monitors?', 'source': 'pv', 'value': 'yes'},\n", " {'name': \"What's the engine's sampling period?\",\n", " 'source': 'pv',\n", " 'value': '0.07'},\n", " {'name': 'The SCAN period (ms) after applying the jitter factor',\n", " 'source': 'pv',\n", " 'value': '0'},\n", " {'name': 'Is this PV currently connected?', 'source': 'pv', 'value': 'yes'},\n", " {'name': 'Connection state at last connection changed event',\n", " 'source': 'pv',\n", " 'value': 'Not connected'},\n", " {'name': 'When did we receive the last event?',\n", " 'source': 'pv',\n", " 'value': 'Oct/18/2024 10:21:48 +02:00'},\n", " {'name': 'What did we last push the data to the short term store?',\n", " 'source': 'pv',\n", " 'value': 'Oct/18/2024 10:21:48 +02:00'},\n", " {'name': 'When did we request CA to make a connection to this PV?',\n", " 'source': 'pv',\n", " 'value': 'Aug/27/2024 10:35:52 +02:00'},\n", " {'name': 'When did we first establish a connection to this PV?',\n", " 'source': 'pv',\n", " 'value': 'Aug/27/2024 10:35:52 +02:00'},\n", " {'name': 'When did we last lose and reestablish a connection to this PV?',\n", " 'source': 'pv',\n", " 'value': 'Sep/12/2024 11:13:11 +02:00'},\n", " {'name': 'When did we last lose a connection to this PV?',\n", " 'source': 'pv',\n", " 'value': 'Never'},\n", " {'name': 'How many times have we lost and regained the connection to this PV?',\n", " 'source': 'pv',\n", " 'value': '8'},\n", " {'name': 'How many events so far?', 'source': 'pv', 'value': '48804684'},\n", " {'name': 'How many raw scan events so far?', 'source': 'pv', 'value': '0'},\n", " {'name': 'How many events lost because the timestamp is in the far future or past so far?',\n", " 'source': 'pv',\n", " 'value': '0'},\n", " {'name': 'Timestamp of last event from the IOC - correct or not.',\n", " 'source': 'pv',\n", " 'value': 'Oct/18/2024 10:21:48 +02:00'},\n", " {'name': 'How many events lost because the sample buffer is full so far?',\n", " 'source': 'pv',\n", " 'value': '69331'},\n", " {'name': 'How many events lost because the DBR_Type of the PV has changed from what it used to be?',\n", " 'source': 'pv',\n", " 'value': '0'},\n", " {'name': 'How many events lost totally so far?',\n", " 'source': 'pv',\n", " 'value': '69331'},\n", " {'name': 'Average bytes per event', 'source': 'pv', 'value': '20.11'},\n", " {'name': 'Estimated event rate (events/sec)',\n", " 'source': 'pv',\n", " 'value': '15.71'},\n", " {'name': 'Estimated storage rate (KB/hour)',\n", " 'source': 'pv',\n", " 'value': '1,110.55'},\n", " {'name': 'Estimated storage rate (MB/day)', 'source': 'pv', 'value': '26.03'},\n", " {'name': 'Estimated storage rate (GB/year)', 'source': 'pv', 'value': '9.28'},\n", " {'name': 'PV connection state machine state',\n", " 'source': 'CA',\n", " 'value': 'GotMonitor'},\n", " {'name': 'Last monitor received at',\n", " 'source': 'CA',\n", " 'value': 'Oct/18/2024 10:21:48 +02:00'},\n", " {'name': 'Last monitor had a valid DBR?', 'source': 'CA', 'value': 'true'},\n", " {'name': 'Last monitor event timestamp',\n", " 'source': 'CA',\n", " 'value': 'Oct/18/2024 10:21:48 +02:00'},\n", " {'name': 'Various transient errors', 'source': 'CA', 'value': '0'},\n", " {'name': 'Do we have a CA channel?', 'source': 'CA', 'value': 'true'},\n", " {'name': 'Do we have a subscription?', 'source': 'CA', 'value': 'true'},\n", " {'name': 'CAJ Searches', 'source': 'CA', 'value': '2'},\n", " {'name': 'CAJ channel ID (CID)', 'source': 'CA', 'value': '506'},\n", " {'name': 'CAJ server channel ID (SID)', 'source': 'CA', 'value': '23301'},\n", " {'name': 'CAJ subscription ID', 'source': 'CA', 'value': '79595'},\n", " {'name': 'CAJ connection state', 'source': 'CA', 'value': 'CONNECTED'},\n", " {'name': 'Daily metadata last saved at', 'source': 'CA', 'value': 'Never'},\n", " {'name': 'Do we use DBE_Properties?', 'source': 'CA', 'value': 'false'},\n", " {'name': 'Do we have a DBE Properties subscription?',\n", " 'source': 'CA',\n", " 'value': 'false'},\n", " {'name': 'The internal connected bool', 'source': 'CA', 'value': 'true'},\n", " {'name': 'The internal running bool', 'source': 'CA', 'value': 'true'},\n", " {'name': 'Do we have a valid DBR Type constructor',\n", " 'source': 'CA',\n", " 'value': 'true'},\n", " {'name': 'The CAJ command thread id', 'source': 'CA', 'value': '3'},\n", " {'name': \"Any other PV's being controlled?\",\n", " 'source': 'CA',\n", " 'value': 'false'},\n", " {'name': 'Has metafields?', 'source': 'CA', 'value': 'true'},\n", " {'name': 'Hostname of PV from CA',\n", " 'source': 'CA',\n", " 'value': 'dtl-cavhl-ipc-ioc.tn.esss.lu.se'},\n", " {'name': 'Channels for the extra fields', 'source': 'pv', 'value': '9'},\n", " {'name': 'Connected channels for the extra fields',\n", " 'source': 'pv',\n", " 'value': '0'},\n", " {'name': 'Sample buffer capacity', 'source': 'pv', 'value': '215'},\n", " {'name': 'Time elapsed since search request (s)',\n", " 'source': 'pv',\n", " 'value': '-1'},\n", " {'name': 'Name (from ETL)',\n", " 'source': 'etl',\n", " 'value': 'DTL-010:EMR-TT-001:Temp'},\n", " {'name': 'ETL 0 partition granularity of source',\n", " 'source': 'etl',\n", " 'value': 'PARTITION_DAY'},\n", " {'name': 'ETL 0 partition granularity of dest',\n", " 'source': 'etl',\n", " 'value': 'PARTITION_MONTH'},\n", " {'name': 'ETL 0 last completed',\n", " 'source': 'etl',\n", " 'value': 'Oct/18/2024 10:14:10 +02:00'},\n", " {'name': 'ETL 0 last job took (ms)', 'source': 'etl', 'value': '127'},\n", " {'name': 'ETL 0 next job runs at',\n", " 'source': 'etl',\n", " 'value': 'Oct/18/2024 18:14:10 +02:00'},\n", " {'name': 'ETL 0 total time performing ETL(ms)',\n", " 'source': 'etl',\n", " 'value': '9599'},\n", " {'name': 'ETL 0 average time performing ETL(ms)',\n", " 'source': 'etl',\n", " 'value': '61'},\n", " {'name': 'ETL 0 number of times we performed ETL',\n", " 'source': 'etl',\n", " 'value': '155'},\n", " {'name': 'ETL 0 out of space chunks deleted', 'source': 'etl', 'value': '0'},\n", " {'name': 'ETL 0 approx bytes transferred(MB)',\n", " 'source': 'etl',\n", " 'value': '1,463.88'},\n", " {'name': 'ETL Total time spent by getETLStreams() in ETL(0) (ms)',\n", " 'source': 'etl',\n", " 'value': '1407'},\n", " {'name': 'ETL Total time spent by free space checks in ETL(0) (ms)',\n", " 'source': 'etl',\n", " 'value': '15'},\n", " {'name': 'ETL Total time spent by prepareForNewPartition() in ETL(0) (ms)',\n", " 'source': 'etl',\n", " 'value': '0'},\n", " {'name': 'ETL Total time spent by appendToETLAppendData() in ETL(0) (ms)',\n", " 'source': 'etl',\n", " 'value': '7902'},\n", " {'name': 'ETL Total time spent by commitETLAppendData() in ETL(0) (ms)',\n", " 'source': 'etl',\n", " 'value': '0'},\n", " {'name': 'ETL Total time spent by markForDeletion() in ETL(0) (ms)',\n", " 'source': 'etl',\n", " 'value': '273'},\n", " {'name': 'ETL Total time spent by runPostProcessors() in ETL(0) (ms)',\n", " 'source': 'etl',\n", " 'value': '0'},\n", " {'name': 'ETL Total time spent by executePostETLTasks() in ETL(0) (ms)',\n", " 'source': 'etl',\n", " 'value': '0'},\n", " {'name': 'ETL 1 partition granularity of source',\n", " 'source': 'etl',\n", " 'value': 'PARTITION_MONTH'},\n", " {'name': 'ETL 1 partition granularity of dest',\n", " 'source': 'etl',\n", " 'value': 'PARTITION_YEAR'},\n", " {'name': 'ETL 1 last completed',\n", " 'source': 'etl',\n", " 'value': 'Oct/18/2024 06:26:05 +02:00'},\n", " {'name': 'ETL 1 last job took (ms)', 'source': 'etl', 'value': '43'},\n", " {'name': 'ETL 1 next job runs at',\n", " 'source': 'etl',\n", " 'value': 'Oct/18/2024 14:26:05 +02:00'},\n", " {'name': 'ETL 1 total time performing ETL(ms)',\n", " 'source': 'etl',\n", " 'value': '31183'},\n", " {'name': 'ETL 1 average time performing ETL(ms)',\n", " 'source': 'etl',\n", " 'value': '219'},\n", " {'name': 'ETL 1 number of times we performed ETL',\n", " 'source': 'etl',\n", " 'value': '142'},\n", " {'name': 'ETL 1 out of space chunks deleted', 'source': 'etl', 'value': '0'},\n", " {'name': 'ETL 1 approx bytes transferred(MB)',\n", " 'source': 'etl',\n", " 'value': '1,637.34'},\n", " {'name': 'ETL Total time spent by getETLStreams() in ETL(1) (ms)',\n", " 'source': 'etl',\n", " 'value': '3541'},\n", " {'name': 'ETL Total time spent by free space checks in ETL(1) (ms)',\n", " 'source': 'etl',\n", " 'value': '68'},\n", " {'name': 'ETL Total time spent by prepareForNewPartition() in ETL(1) (ms)',\n", " 'source': 'etl',\n", " 'value': '0'},\n", " {'name': 'ETL Total time spent by appendToETLAppendData() in ETL(1) (ms)',\n", " 'source': 'etl',\n", " 'value': '27572'},\n", " {'name': 'ETL Total time spent by commitETLAppendData() in ETL(1) (ms)',\n", " 'source': 'etl',\n", " 'value': '0'},\n", " {'name': 'ETL Total time spent by markForDeletion() in ETL(1) (ms)',\n", " 'source': 'etl',\n", " 'value': '1'},\n", " {'name': 'ETL Total time spent by runPostProcessors() in ETL(1) (ms)',\n", " 'source': 'etl',\n", " 'value': '0'},\n", " {'name': 'ETL Total time spent by executePostETLTasks() in ETL(1) (ms)',\n", " 'source': 'etl',\n", " 'value': '0'}]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "archiver.get_pv_details(pv)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Getting Data" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946776, year=2024, nanos=871630988, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946776, year=2024, nanos=911302087, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.11, secondsintoyear=29946776, year=2024, nanos=961295853, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946777, year=2024, nanos=11289178, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=51297845, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946777, year=2024, nanos=151318914, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=191285978, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=241294108, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=291292505, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.060000000000002, secondsintoyear=29946777, year=2024, nanos=331289395, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=381343840, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946777, year=2024, nanos=471308416, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=521299898, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.12, secondsintoyear=29946777, year=2024, nanos=571315402, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=611308354, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=661290527, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=711283161, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=851289542, severity=0, status=0, field_values=[]),\n", " ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.05, secondsintoyear=29946777, year=2024, nanos=891281183, severity=0, status=0, field_values=[])]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "archiver.get_events(pv, datetime.now(tz=UTC) - timedelta(seconds=1), datetime.now(tz=UTC))" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "df = archiver.get_data(pv, datetime.now(tz=UTC) - timedelta(seconds=30), datetime.now(tz=UTC))" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | val | \n", "
|---|---|
| date | \n", "\n", " |
| 2024-12-12 14:32:29.221293335+00:00 | \n", "25.07 | \n", "
| 2024-12-12 14:32:29.271292973+00:00 | \n", "25.11 | \n", "
| 2024-12-12 14:32:29.321302366+00:00 | \n", "25.07 | \n", "
| 2024-12-12 14:32:29.361298308+00:00 | \n", "25.09 | \n", "
| 2024-12-12 14:32:29.411294264+00:00 | \n", "25.11 | \n", "