Asyn + Streamdevice: Gamma Vacuum Digtial QPC, missmatch after some time

["Dr. Simon Friederich via Tech-talk" <tech-talk at aps.anl.gov>]

Dear EPICS-community,

I am struggeling with our implementation of the IGP controller from Gamma Vacuum Digital QPC that we use.
Out of the sudden or at a certain point, which I can neither explain nor reproduce, the IOC complains that the controllers' responses to the protocol no longer fit.

Here is an example output that I've retrieved in the morning, before that the IOC ran for weeks without error:

2024/10/08 08:15:41.970931 igpctrlmodh2071_port melba060:pump:i_get_raw: No reply within 2000 ms to "cmd 0A 1<0d>"
2024/10/08 08:15:43.974231 igpctrlmodh2071_port melba060:pump:p_get_raw: No reply within 2000 ms to "cmd 0B 1<0d>"
2024/10/08 08:15:45.977001 igpctrlmodh2071_port melba060:pump:u_get: No reply within 2000 ms to "cmd 0C 1<0d>"
2024/10/08 08:15:46.722205 igpctrlmodh2071_port melba040:diffp_down:on_get: Input "1.3E-11 MBAR" does not match format "%{NO|YES}"
2024/10/08 08:15:46.842189 igpctrlmodh2071_port melba060:pump:stat_get: Input "13" does not match format "%{STANDBY|RUNNING 00|SAFE-CONN 00|INTERLOCK 00|PUMP ERROR 00|CALIBRATION 00}"
2024/10/08 08:15:46.967906 igpctrlmodh2071_port melba040:diffp_down:stat_get: Input "18" does not match format "%{STANDBY|RUNNING 00|SAFE-CONN 00|INTERLOCK 00|PUMP ERROR 00|CALIBRATION 00}"
2024/10/08 08:15:47.087939 igpctrlmodh2071_port melba040:diffp_down:i_get_raw: Input "NO" does not match format "%e"
2024/10/08 08:15:47.207884 igpctrlmodh2071_port melba040:diffp_down:p_get_raw: Input "STANDBY" does not match format "%e"
2024/10/08 08:15:47.327939 igpctrlmodh2071_port melba060:pump:on_get: Input "PUMP ERROR 00" does not match format "%{NO|YES}"
2024/10/08 08:15:47.693674 igpctrlmodh2071_port melba060:pump:u_get: Input "NO" does not match format "%i"
2024/10/08 08:15:48.059375 igpctrlmodh2071_port melba040:diffp_down:i_get_raw: Input "OK 00 14" too short.
2024/10/08 08:15:48.059449 igpctrlmodh2071_port No match for " AMPS"
2024/10/08 08:15:48.179663 igpctrlmodh2071_port melba040:diffp_down:p_get_raw: Input "OK 00 7.5E-11 AMPS"
2024/10/08 08:15:48.179734 igpctrlmodh2071_port melba040:diffp_down:p_get_raw: mismatch after 14 bytes "...0 7.5E-11 "
2024/10/08 08:15:48.179764 igpctrlmodh2071_port melba040:diffp_down:p_get_raw: got "AMPS" where "MBAR" was expected
2024/10/08 08:15:48.299339 igpctrlmodh2071_port melba060:pump:on_get: Input "1.3E-11 MBAR" does not match format "%{NO|YES}"
2024/10/08 08:15:48.419330 igpctrlmodh2071_port melba060:pump:i_get_raw: Input "NO" does not match format "%e"
2024/10/08 08:15:48.545033 igpctrlmodh2071_port melba060:pump:p_get_raw: Input "OK 00 1.9E-10 AMPS"
2024/10/08 08:15:48.545094 igpctrlmodh2071_port melba060:pump:p_get_raw: mismatch after 14 bytes "...0 1.9E-10 "
2024/10/08 08:15:48.545132 igpctrlmodh2071_port melba060:pump:p_get_raw: got "AMPS" where "MBAR" was expected
2024/10/08 08:16:35.567653 igpctrlmodh2071_port melba040:diffp_down:p_get_raw: No reply within 2000 ms to "cmd 0B 3<0d>"
2024/10/08 08:16:36.821755 igpctrlmodh2071_port melba060:pump:on_get: Input "1.3E-11 MBAR" does not match format "%{NO|YES}"

I attach the proto and database files we've written. Maybe you can spot any obvious error or have any advice for me to further debug this issue.

Best regards,
Simon

-- 
Dr. Simon Friederich
Institut für Kernphysik
Kollaboration B | MESA
Johannes Gutenberg-Universität Mainz
Johann-Joachim-Becher-Weg 45
D 55128 Mainz

Tel.: +49 (0)6131 39-23160
E-Mail: s.friederich at uni-mainz.de
  

OutTerminator = CR;
InTerminator = CR CR LF ">";

replytimeout = 2000; # 500 ms for the device and another 500 ms for network latency
#readtimeout = 2000;
extrainput = Ignore;

setPressureUnit {
    out "cmd 0E M";
    in "%*?[>]OK 00 ";
    @init {
        # set pressure units to mbar
        out "cmd 0E M";
        in "%*?[>]OK 00 ";
    }
}

debug {
    out "%s";
    in "%39c";
}

modelNumber {
    out "cmd 01";
    in "%*?[>]OK 00 %39c";
}

firmwareVersion {
    out "cmd 02";
    in "%*?[>]OK 00 %39c";
}

getHVStatus {
    out "cmd 61 \$1";
    in "%*?[>]OK 00 %{NO|YES}";
}


rdCurrent {
    out "cmd 0A \$1";
    in "%*?[>]OK 00 %e AMPS";
}

# Neu
rdPressure {
    out "cmd 0B \$1";
    in "%*?[>]OK 00 %e MBAR";
}

rdVoltage {
    out "cmd 0C \$1";
    in "%*?[>]OK 00 %i";
}

rdPumpStatus {
    out "cmd 0D \$1";
    in "%*?[>]OK 00 %{STANDBY|RUNNING 00|SAFE-CONN 00|INTERLOCK 00|PUMP ERROR 00|CALIBRATION 00}";
}

startPump {
    out "cmd %{38|37} \$1";
    in "%*?[>]OK 00 ";
    @mismatch{ in "%*?[>]OK 00 %{*ERROR: PUMP IS ON|*ERROR: PUMP CANNOT START}";}
    @init{
        out "cmd 61 \$1";
        in "%*?[>]OK 00 %{NO|YES}";
    }
}

rdPumpSize {
    out "cmd 11 \$1";
    in "%*?[>]OK 00 %i L/S";
}

setPumpSize {
    out "cmd 12 \$1,%i";
    in "%*?[>]OK 00";
}

rdPumpName {
    out "cmd ED \$1";
    in "%*?[>]OK 00 %15c";
}

setPumpName {
    out "cmd ED \$1,%s";
    in "%*?[>]OK 00";
}

rdSetPoint {
    out "cmd 3B \$1";
    in "%*?[>]OK 00 %s";
}

# Data: N,M,S,X.XE-XX,Y.YE-YY
# Where:
# N is the set-point number (1-8)
# M is the function 0=Off, 1=Pressure Setpoint, 2=HV Error, 3=HV On/Off Indicator
# S is the configured supply (1, ,2 ,3 or 4)
#X.XE-XX is the On Pressure
#Y.YE-YY is the Off Pressure*
# (only rdSetPoint) A indicates: 1 = Relay/TTL output energized, 0 = Relay/TTL output de-energized)
# *Off Pressure must be at least 20% greater than On Pressure.
# If not, controller will automatically set Off
# Pressure to 20% greater than On Pressure.
setSetPoint {
    out "cmd 3B %s";
    in "%*?[>]OK 00 %s";
}

record(bi, "$(P)$(R)on_get") {
    field(DESC, "Reads on/off status for supply CH$(CH)")
    field(DTYP, "stream")
    field(INP, "@gv4qpc.proto getHVStatus($(CH)) $(PORT)")
    field(SCAN, "5 second")
    field(ZNAM, "OFF")
    field(ONAM, "ON")
    field(FLNK, "$(P)$(R)p_get")
    field(FLNK, "$(P)$(R)i_get")
}

record(bo, "$(P)$(R)on_set") {
    field(DESC, "Turn on Supply CH$(CH)")
    field(DTYP, "stream")
    field(OUT, "@gv4qpc.proto startPump($(CH)) $(PORT)")
    field(ZNAM, "OFF")
    field(ONAM, "ON")
}

record(ai, "$(P)$(R)i_get_raw") {
    field(DESC, "Reads current raw for supply CH$(CH)")
    field(DTYP, "stream")
    field(INP, "@gv4qpc.proto rdCurrent($(CH)) $(PORT)")
    field(SCAN, "1 second")
    field(EGU, "A")
    field(PREC, "2")
    field(FLNK, "$(P)$(R)i_get")
}

record(calc, "$(P)$(R)i_get") {
    field(DESC, "Reads current for supply CH$(CH)")
    field(INPA, "$(P)$(R)on_get")
    field(INPB, "$(P)$(R)i_get_raw")
    field(CALC, "A==1?B:0")
    field(EGU, "A")
    field(HOPR, "1e-5")
    field(LOPR, "1e-10")
    field(PREC, "2")
}

record(ai, "$(P)$(R)p_get_raw") {
    field(DESC, "Reads pressure raw for supply CH$(CH)")
    field(DTYP, "stream")
    field(INP, "@gv4qpc.proto rdPressure($(CH)) $(PORT)")
    field(SCAN, "1 second")
    field(EGU, "mbar")
    field(PREC, "2")
    field(FLNK, "$(P)$(R)p_get")
}

record(calc, "$(P)$(R)p_get") {
    field(DESC, "Reads pressure for supply CH$(CH)")
    field(INPA, "$(P)$(R)on_get")
    field(INPB, "$(P)$(R)p_get_raw")
    field(CALC, "A==1?B:0")
    field(EGU, "mbar")
    field(HOPR, "1e-3")
    field(LOPR, "1e-12")
    field(PREC, "2")
}

record(ai, "$(P)$(R)u_get") {
    field(DESC, "Reads voltage for supply CH$(CH)")
    field(DTYP, "stream")
    field(INP, "@gv4qpc.proto rdVoltage($(CH)) $(PORT)")
    field(SCAN, "2 second")
    field(EGU, "V")
    field(LOPR, "0")
    field(HOPR, "7000")
    field(PREC, 0)
}

record(ai, "$(P)$(R)size_get") {
    field(DESC, "Reads pump size for supply CH$(CH)")
    field(DTYP, "stream")
    field(INP, "@gv4qpc.proto rdPumpSize($(CH)) $(PORT)")
    field(EGU, "L/S")
    field(LOPR, "1")
    field(HOPR, "1200")
    field(PREC, "0")
    field(PINI, "YES")
}

record(ao, "$(P)$(R)size_set") {
    field(DESC, "Reads pump size for supply CH$(CH)")
    field(DTYP, "stream")
    field(OUT, "@gv4qpc.proto setPumpSize($(CH)) $(PORT)")
    field(EGU, "L/S")
    field(DRVL, "1")
    field(DRVH, "1200")
    field(PREC, "0")
    field(DISP, 1)
}

record(stringin, "$(P)$(R)nam_get") {
    field(DESC, "Read pump name for supply CH$(CH)")
    field(DTYP, "stream")
    field(INP, "@gv4qpc.proto rdPumpName($(CH)) $(PORT)")
    field(PINI, "YES")
}

record(stringout, "$(P)$(R)nam_set") {
    field(DESC, "Set pump name for supply CH$(CH)")
    field(DTYP, "stream")
    field(OUT, "@gv4qpc.proto setPumpName($(CH)) $(PORT)")
}

record (mbbi, "$(P)$(R)stat_get")
{
    field(DESC, "Show pump status CH$(CH)")
    field(DTYP, "stream")
    field(INP,  "@gv4qpc.proto rdPumpStatus($(CH)) $(PORT)")
    field(SCAN, "10 second")
    field(ZRST, "standby")
    field(ZRVL, "0")
    field(ONST, "running")
    field(ONVL, "1")
    field(TWST, "safe-conn")
    field(TWVL, "2")
    field(THST, "interlock")
    field(THVL, "3")
    field(FRST, "pump error")
    field(FRVL, "4")
	field(FVST, "calibration")
	field(FVVL, "5")
}

record(stringout, "$(P)$(R)debug") {
    field(DTYP, "stream")
    field(OUT, "@gv4qpc.proto debug $(PORT)")
}

record(stringout, "$(P)$(R)unit_set") {
    field(DTYP, "stream")
    field(OUT, "@gv4qpc.proto setPressureUnit $(PORT)")
    field(DISP, 1)
}

record(stringin, "$(P)$(R)idn_get") {
    field(DESC, "A description of the controller")
    field(DTYP, "stream")
    field(INP, "@gv4qpc.proto modelNumber $(PORT)")
    field(SCAN, "Passive")
    field(PINI, "YES")
    #field(SDIS, "$(P)$(R)deviceEnabled.RVAL")
}

record(stringin, "$(P)$(R)firmware_get") {
    field(DESC, "Firmware revision level")
    field(DTYP, "stream")
    field(INP, "@gv4qpc.proto firmwareVersion $(PORT)")
    field(SCAN, "Passive")
    field(PINI, "YES")
    #field(SDIS, "$(P)$(R)deviceEnabled.RVAL")
}