Library Examples

Archiving information

from datetime import datetime, timedelta

from pytz import UTC

from epicsarchiver import ArchiverAppliance

%matplotlib inline

archiver = ArchiverAppliance("archiver-linac-04.tn.esss.lu.se")
pv = "DTL-010:EMR-TT-001:Temp"

archiver.get_pv_status(pv)

[{'lastRotateLogs': 'Never',
  'appliance': 'archiver-linac-02',
  'pvName': 'DTL-010:EMR-TT-001:Temp',
  'pvNameOnly': 'DTL-010:EMR-TT-001:Temp',
  'connectionState': 'true',
  'lastEvent': 'Dec/12/2024 15:32:52 +01:00',
  'samplingPeriod': '0.07',
  'isMonitored': 'true',
  'connectionLastRestablished': 'Sep/12/2024 11:13:11 +02:00',
  'connectionFirstEstablished': 'Aug/27/2024 10:35:52 +02:00',
  'connectionLossRegainCount': '8',
  'status': 'Being archived'}]

archiver.get_pv_details(pv)

Show code cell output

Hide code cell output

[{'name': 'PV Name', 'value': 'DTL-010:EMR-TT-001:Temp', 'source': 'mgmt'},
 {'name': 'Instance archiving PV',
  'value': 'archiver-linac-02',
  'source': 'mgmt'},
 {'name': 'Archival params creation time:',
  'value': 'Mar/09/2023 10:11:56 +01:00',
  'source': 'mgmt'},
 {'name': 'Archival params modification time:',
  'value': 'Mar/10/2023 10:00:41 +01:00',
  'source': 'mgmt'},
 {'name': 'Archiver DBR type (from typeinfo):',
  'value': 'DBR_SCALAR_DOUBLE',
  'source': 'mgmt'},
 {'name': 'Is this a scalar:', 'value': 'Yes', 'source': 'mgmt'},
 {'name': 'Number of elements:', 'value': '1', 'source': 'mgmt'},
 {'name': 'Precision:', 'value': '0.0', 'source': 'mgmt'},
 {'name': 'Units:', 'value': 'degC', 'source': 'mgmt'},
 {'name': 'Is this PV paused:', 'value': 'No', 'source': 'mgmt'},
 {'name': 'Sampling method:', 'value': 'MONITOR', 'source': 'mgmt'},
 {'name': 'Sampling period:', 'value': '0.07', 'source': 'mgmt'},
 {'name': 'Are we using PVAccess?', 'value': 'No', 'source': 'mgmt'},
 {'name': 'Extra info - MDEL:', 'value': '0.0', 'source': 'mgmt'},
 {'name': 'Extra info - ADEL:', 'value': '0.0', 'source': 'mgmt'},
 {'name': 'Extra info - SCAN:', 'value': 'Passive', 'source': 'mgmt'},
 {'name': 'Extra info - RTYP:', 'value': 'ai', 'source': 'mgmt'},
 {'name': 'Archive Fields',
  'value': 'EGU,HIHI,HIGH,ADEL,LOW,LOLO,LOPR,HOPR',
  'source': 'mgmt'},
 {'name': 'Channel Name', 'source': 'pv', 'value': 'DTL-010:EMR-TT-001:Temp'},
 {'name': 'Host name',
  'source': 'pv',
  'value': 'dtl-cavhl-ipc-ioc.tn.esss.lu.se'},
 {'name': 'Controlling PV', 'source': 'pv', 'value': ''},
 {'name': 'Is engine currently archiving this?',
  'source': 'pv',
  'value': 'yes'},
 {'name': 'Archiver DBR type (initial)',
  'source': 'pv',
  'value': 'DBR_SCALAR_DOUBLE'},
 {'name': 'Archiver DBR type (from CA)',
  'source': 'pv',
  'value': 'DBR_SCALAR_DOUBLE'},
 {'name': 'Number of elements per event (from CA)',
  'source': 'pv',
  'value': '1'},
 {'name': 'Is engine using monitors?', 'source': 'pv', 'value': 'yes'},
 {'name': "What's the engine's sampling period?",
  'source': 'pv',
  'value': '0.07'},
 {'name': 'The SCAN period (ms) after applying the jitter factor',
  'source': 'pv',
  'value': '0'},
 {'name': 'Is this PV currently connected?', 'source': 'pv', 'value': 'yes'},
 {'name': 'Connection state at last connection changed event',
  'source': 'pv',
  'value': 'Not connected'},
 {'name': 'When did we receive the last event?',
  'source': 'pv',
  'value': 'Oct/18/2024 10:21:48 +02:00'},
 {'name': 'What did we last push the data to the short term store?',
  'source': 'pv',
  'value': 'Oct/18/2024 10:21:48 +02:00'},
 {'name': 'When did we request CA to make a connection to this PV?',
  'source': 'pv',
  'value': 'Aug/27/2024 10:35:52 +02:00'},
 {'name': 'When did we first establish a connection to this PV?',
  'source': 'pv',
  'value': 'Aug/27/2024 10:35:52 +02:00'},
 {'name': 'When did we last lose and reestablish a connection to this PV?',
  'source': 'pv',
  'value': 'Sep/12/2024 11:13:11 +02:00'},
 {'name': 'When did we last lose a connection to this PV?',
  'source': 'pv',
  'value': 'Never'},
 {'name': 'How many times have we lost and regained the connection to this PV?',
  'source': 'pv',
  'value': '8'},
 {'name': 'How many events so far?', 'source': 'pv', 'value': '48804684'},
 {'name': 'How many raw scan events so far?', 'source': 'pv', 'value': '0'},
 {'name': 'How many events lost because the timestamp is in the far future or past so far?',
  'source': 'pv',
  'value': '0'},
 {'name': 'Timestamp of last event from the IOC - correct or not.',
  'source': 'pv',
  'value': 'Oct/18/2024 10:21:48 +02:00'},
 {'name': 'How many events lost because the sample buffer is full so far?',
  'source': 'pv',
  'value': '69331'},
 {'name': 'How many events lost because the DBR_Type of the PV has changed from what it used to be?',
  'source': 'pv',
  'value': '0'},
 {'name': 'How many events lost totally so far?',
  'source': 'pv',
  'value': '69331'},
 {'name': 'Average bytes per event', 'source': 'pv', 'value': '20.11'},
 {'name': 'Estimated event rate (events/sec)',
  'source': 'pv',
  'value': '15.71'},
 {'name': 'Estimated storage rate (KB/hour)',
  'source': 'pv',
  'value': '1,110.55'},
 {'name': 'Estimated storage rate (MB/day)', 'source': 'pv', 'value': '26.03'},
 {'name': 'Estimated storage rate (GB/year)', 'source': 'pv', 'value': '9.28'},
 {'name': 'PV connection state machine state',
  'source': 'CA',
  'value': 'GotMonitor'},
 {'name': 'Last monitor received at',
  'source': 'CA',
  'value': 'Oct/18/2024 10:21:48 +02:00'},
 {'name': 'Last monitor had a valid DBR?', 'source': 'CA', 'value': 'true'},
 {'name': 'Last monitor event timestamp',
  'source': 'CA',
  'value': 'Oct/18/2024 10:21:48 +02:00'},
 {'name': 'Various transient errors', 'source': 'CA', 'value': '0'},
 {'name': 'Do we have a CA channel?', 'source': 'CA', 'value': 'true'},
 {'name': 'Do we have a subscription?', 'source': 'CA', 'value': 'true'},
 {'name': 'CAJ Searches', 'source': 'CA', 'value': '2'},
 {'name': 'CAJ channel ID (CID)', 'source': 'CA', 'value': '506'},
 {'name': 'CAJ server channel ID (SID)', 'source': 'CA', 'value': '23301'},
 {'name': 'CAJ subscription ID', 'source': 'CA', 'value': '79595'},
 {'name': 'CAJ connection state', 'source': 'CA', 'value': 'CONNECTED'},
 {'name': 'Daily metadata last saved at', 'source': 'CA', 'value': 'Never'},
 {'name': 'Do we use DBE_Properties?', 'source': 'CA', 'value': 'false'},
 {'name': 'Do we have a DBE Properties subscription?',
  'source': 'CA',
  'value': 'false'},
 {'name': 'The internal connected bool', 'source': 'CA', 'value': 'true'},
 {'name': 'The internal running bool', 'source': 'CA', 'value': 'true'},
 {'name': 'Do we have a valid DBR Type constructor',
  'source': 'CA',
  'value': 'true'},
 {'name': 'The CAJ command thread id', 'source': 'CA', 'value': '3'},
 {'name': "Any other PV's being controlled?",
  'source': 'CA',
  'value': 'false'},
 {'name': 'Has metafields?', 'source': 'CA', 'value': 'true'},
 {'name': 'Hostname of PV from CA',
  'source': 'CA',
  'value': 'dtl-cavhl-ipc-ioc.tn.esss.lu.se'},
 {'name': 'Channels for the extra fields', 'source': 'pv', 'value': '9'},
 {'name': 'Connected channels for the extra fields',
  'source': 'pv',
  'value': '0'},
 {'name': 'Sample buffer capacity', 'source': 'pv', 'value': '215'},
 {'name': 'Time elapsed since search request (s)',
  'source': 'pv',
  'value': '-1'},
 {'name': 'Name (from ETL)',
  'source': 'etl',
  'value': 'DTL-010:EMR-TT-001:Temp'},
 {'name': 'ETL 0 partition granularity of source',
  'source': 'etl',
  'value': 'PARTITION_DAY'},
 {'name': 'ETL 0 partition granularity of dest',
  'source': 'etl',
  'value': 'PARTITION_MONTH'},
 {'name': 'ETL 0 last completed',
  'source': 'etl',
  'value': 'Oct/18/2024 10:14:10 +02:00'},
 {'name': 'ETL 0 last job took (ms)', 'source': 'etl', 'value': '127'},
 {'name': 'ETL 0 next job runs at',
  'source': 'etl',
  'value': 'Oct/18/2024 18:14:10 +02:00'},
 {'name': 'ETL 0 total time performing ETL(ms)',
  'source': 'etl',
  'value': '9599'},
 {'name': 'ETL 0 average time performing ETL(ms)',
  'source': 'etl',
  'value': '61'},
 {'name': 'ETL 0 number of times we performed ETL',
  'source': 'etl',
  'value': '155'},
 {'name': 'ETL 0 out of space chunks deleted', 'source': 'etl', 'value': '0'},
 {'name': 'ETL 0 approx bytes transferred(MB)',
  'source': 'etl',
  'value': '1,463.88'},
 {'name': 'ETL Total time spent by getETLStreams() in ETL(0) (ms)',
  'source': 'etl',
  'value': '1407'},
 {'name': 'ETL Total time spent by free space checks in ETL(0) (ms)',
  'source': 'etl',
  'value': '15'},
 {'name': 'ETL Total time spent by prepareForNewPartition() in ETL(0) (ms)',
  'source': 'etl',
  'value': '0'},
 {'name': 'ETL Total time spent by appendToETLAppendData() in ETL(0) (ms)',
  'source': 'etl',
  'value': '7902'},
 {'name': 'ETL Total time spent by commitETLAppendData() in ETL(0) (ms)',
  'source': 'etl',
  'value': '0'},
 {'name': 'ETL Total time spent by markForDeletion() in ETL(0) (ms)',
  'source': 'etl',
  'value': '273'},
 {'name': 'ETL Total time spent by runPostProcessors() in ETL(0) (ms)',
  'source': 'etl',
  'value': '0'},
 {'name': 'ETL Total time spent by executePostETLTasks() in ETL(0) (ms)',
  'source': 'etl',
  'value': '0'},
 {'name': 'ETL 1 partition granularity of source',
  'source': 'etl',
  'value': 'PARTITION_MONTH'},
 {'name': 'ETL 1 partition granularity of dest',
  'source': 'etl',
  'value': 'PARTITION_YEAR'},
 {'name': 'ETL 1 last completed',
  'source': 'etl',
  'value': 'Oct/18/2024 06:26:05 +02:00'},
 {'name': 'ETL 1 last job took (ms)', 'source': 'etl', 'value': '43'},
 {'name': 'ETL 1 next job runs at',
  'source': 'etl',
  'value': 'Oct/18/2024 14:26:05 +02:00'},
 {'name': 'ETL 1 total time performing ETL(ms)',
  'source': 'etl',
  'value': '31183'},
 {'name': 'ETL 1 average time performing ETL(ms)',
  'source': 'etl',
  'value': '219'},
 {'name': 'ETL 1 number of times we performed ETL',
  'source': 'etl',
  'value': '142'},
 {'name': 'ETL 1 out of space chunks deleted', 'source': 'etl', 'value': '0'},
 {'name': 'ETL 1 approx bytes transferred(MB)',
  'source': 'etl',
  'value': '1,637.34'},
 {'name': 'ETL Total time spent by getETLStreams() in ETL(1) (ms)',
  'source': 'etl',
  'value': '3541'},
 {'name': 'ETL Total time spent by free space checks in ETL(1) (ms)',
  'source': 'etl',
  'value': '68'},
 {'name': 'ETL Total time spent by prepareForNewPartition() in ETL(1) (ms)',
  'source': 'etl',
  'value': '0'},
 {'name': 'ETL Total time spent by appendToETLAppendData() in ETL(1) (ms)',
  'source': 'etl',
  'value': '27572'},
 {'name': 'ETL Total time spent by commitETLAppendData() in ETL(1) (ms)',
  'source': 'etl',
  'value': '0'},
 {'name': 'ETL Total time spent by markForDeletion() in ETL(1) (ms)',
  'source': 'etl',
  'value': '1'},
 {'name': 'ETL Total time spent by runPostProcessors() in ETL(1) (ms)',
  'source': 'etl',
  'value': '0'},
 {'name': 'ETL Total time spent by executePostETLTasks() in ETL(1) (ms)',
  'source': 'etl',
  'value': '0'}]

Getting Data

archiver.get_events(pv, datetime.now(tz=UTC) - timedelta(seconds=1), datetime.now(tz=UTC))

[ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946776, year=2024, nanos=871630988, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946776, year=2024, nanos=911302087, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.11, secondsintoyear=29946776, year=2024, nanos=961295853, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946777, year=2024, nanos=11289178, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=51297845, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946777, year=2024, nanos=151318914, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=191285978, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=241294108, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=291292505, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.060000000000002, secondsintoyear=29946777, year=2024, nanos=331289395, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=381343840, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946777, year=2024, nanos=471308416, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=521299898, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.12, secondsintoyear=29946777, year=2024, nanos=571315402, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=611308354, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=661290527, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.080000000000002, secondsintoyear=29946777, year=2024, nanos=711283161, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946777, year=2024, nanos=851289542, severity=0, status=0, field_values=[]),
 ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.05, secondsintoyear=29946777, year=2024, nanos=891281183, severity=0, status=0, field_values=[])]

df = archiver.get_data(pv, datetime.now(tz=UTC) - timedelta(seconds=30), datetime.now(tz=UTC))

df.head()

	val
date
2024-12-12 14:32:29.221293335+00:00	25.07
2024-12-12 14:32:29.271292973+00:00	25.11
2024-12-12 14:32:29.321302366+00:00	25.07
2024-12-12 14:32:29.361298308+00:00	25.09
2024-12-12 14:32:29.411294264+00:00	25.11

Async Fetch Data

from epicsarchiver.retrieval.archiver_retrieval.async_archiver_retrieval import AsyncArchiverRetrieval
from pytz import timezone

tz = timezone("Europe/Stockholm")

async with AsyncArchiverRetrieval(archiver.hostname) as a_archiver:
    print(await a_archiver.get_events(pv, datetime.now(tz=tz) - timedelta(microseconds=100), datetime.now(tz=tz)))

[ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.11, secondsintoyear=29946798, year=2024, nanos=201295837, severity=0, status=0, field_values=[]), ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.07, secondsintoyear=29946808, year=2024, nanos=11298347, severity=0, status=0, field_values=[])]

async with AsyncArchiverRetrieval(archiver.hostname) as a_archiver:
    print(await a_archiver.get_all_events([pv, "DTL-020:EMR-TT-001:Temp", "DTL-030:EMR-TT-001:Temp"], datetime.now(tz=tz) - timedelta(microseconds=100), datetime.now(tz=tz)))

{'DTL-010:EMR-TT-001:Temp': [ArchiveEvent(pv='DTL-010:EMR-TT-001:Temp', val=25.09, secondsintoyear=29946810, year=2024, nanos=391296875, severity=0, status=0, field_values=[])], 'DTL-020:EMR-TT-001:Temp': [ArchiveEvent(pv='DTL-020:EMR-TT-001:Temp', val=25.240000000000002, secondsintoyear=29946810, year=2024, nanos=391551705, severity=0, status=0, field_values=[])], 'DTL-030:EMR-TT-001:Temp': [ArchiveEvent(pv='DTL-030:EMR-TT-001:Temp', val=25.28, secondsintoyear=29946810, year=2024, nanos=321705751, severity=0, status=0, field_values=[]), ArchiveEvent(pv='DTL-030:EMR-TT-001:Temp', val=25.3, secondsintoyear=29946810, year=2024, nanos=422025493, severity=0, status=0, field_values=[])]}

Displaying and Calculating Summaries

df.plot()

<Axes: xlabel='date'>

../_images/cf117cc16e98e1e2fd36e4f3b9d53a42aa7a0f25b65f38d35484d908fab128a1.png

from epicsarchiver.retrieval.archiver_retrieval import Processor, ProcessorName

archiver.get_data(pv, datetime.now(tz=UTC) - timedelta(seconds=6000), datetime.now(tz=UTC), Processor(ProcessorName.MEAN, 20)).plot()

<Axes: xlabel='date'>

../_images/821d6bea950de4a381c6392616e8e66bf1e3fe9fba76f669597932f1c030afa9.png