[ Feb  6 17:04:34 Enabled. ]
[ Feb  6 17:04:34 Rereading configuration. ]
[ Feb  6 17:04:39 Rereading configuration. ]
[ Feb  6 17:04:45 Executing start method ("/opt/oxide/lib/svc/manifest/crucible/agent.sh"). ]
+ . /lib/svc/share/smf_include.sh
++ SMF_EXIT_OK=0
++ SMF_EXIT_NODAEMON=94
++ SMF_EXIT_ERR_FATAL=95
++ SMF_EXIT_ERR_CONFIG=96
++ SMF_EXIT_MON_DEGRADE=97
++ SMF_EXIT_MON_OFFLINE=98
++ SMF_EXIT_ERR_NOSMF=99
++ SMF_EXIT_ERR_PERM=100
++ svcprop -c -p config/datalink svc:/oxide/crucible/agent:default
+ DATALINK=oxControlService19
++ svcprop -c -p config/gateway svc:/oxide/crucible/agent:default
+ GATEWAY=fd00:1122:3344:101::1
++ svcprop -c -p config/dataset svc:/oxide/crucible/agent:default
+ DATASET=oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible
++ svcprop -c -p config/listen_addr svc:/oxide/crucible/agent:default
+ LISTEN_ADDR=fd00:1122:3344:101::14
++ svcprop -c -p config/listen_port svc:/oxide/crucible/agent:default
+ LISTEN_PORT=32345
++ svcprop -c -p config/portbase svc:/oxide/crucible/agent:default
+ PORTBASE=19000
++ svcprop -c -p config/downstairs_prefix svc:/oxide/crucible/agent:default
+ DOWNSTAIRS_PREFIX=downstairs
++ svcprop -c -p config/snapshot_prefix svc:/oxide/crucible/agent:default
+ SNAPSHOT_PREFIX=snapshot
+ [[ oxControlService19 == unknown ]]
+ [[ fd00:1122:3344:101::1 == unknown ]]
+ ipadm delete-if oxControlService19
ipadm: Could not delete oxControlService19: Interface does not exist
+ true
+ ipadm create-if -t oxControlService19
+ ipadm set-ifprop -t -p mtu=9000 -m ipv4 oxControlService19
+ ipadm set-ifprop -t -p mtu=9000 -m ipv6 oxControlService19
+ ipadm show-addr oxControlService19/ll
ipadm: Address object not found
+ ipadm create-addr -t -T addrconf oxControlService19/ll
+ ipadm show-addr oxControlService19/omicron6
ipadm: Address object not found
+ ipadm create-addr -t -T static -a fd00:1122:3344:101::14 oxControlService19/omicron6
+ route get -inet6 default -inet6 fd00:1122:3344:101::1
default: not in table
+ route add -inet6 default -inet6 fd00:1122:3344:101::1
add net default: gateway fd00:1122:3344:101::1
+ args=('-D' '/opt/oxide/crucible/bin/crucible-downstairs' '--dataset' "$DATASET" '-l' "[$LISTEN_ADDR]:$LISTEN_PORT" '-P' "$PORTBASE" '-p' "$DOWNSTAIRS_PREFIX" '-s' "$SNAPSHOT_PREFIX")
+ exec /opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible -l '[fd00:1122:3344:101::14]:32345' -P 19000 -p downstairs -s snapshot
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:04:46.310730255Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:04:46.315519079Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:04:46.315529309Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:04:46.46119686Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.617076142Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45565"}
{"msg":"region 8c0d83a5-d472-4072-bac6-8e44a67c7b15 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.689109625Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.692525411Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.692550071Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions","method":"POST","req_id":"332a620d-244e-4a32-b157-ccdbc2b37eb3","remote_addr":"[fd00:1122:3344:101::a]:45565","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":9707,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.703905278Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.703933949Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8c0d83a5-d472-4072-bac6-8e44a67c7b15\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.769887113Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"region":"8c0d83a5-d472-4072-bac6-8e44a67c7b15","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.913104706Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"region":"8c0d83a5-d472-4072-bac6-8e44a67c7b15","component":"worker"}
{"msg":"region 8c0d83a5-d472-4072-bac6-8e44a67c7b15 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.913152877Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.91326014Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.913678202Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.91717505Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.927028235Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.930224584Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.930967425Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.934856334Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.934891175Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.934937466Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.940473041Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:23.944274917Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:06:24.036700321Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions","method":"POST","req_id":"65ad990e-d3af-43d3-9e9e-4982b27f2385","remote_addr":"[fd00:1122:3344:101::a]:45565","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.440709606Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34710"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.440818529Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","method":"GET","req_id":"6571b5f9-e235-491e-b1a2-0c2b6bde1e9d","remote_addr":"[fd00:1122:3344:101::a]:34710","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"path is \"/data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.44762412Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.455941832Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"snapshot is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15@7b23de1a-c7ea-4c85-9c25-9375bdee1c71","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.463060751Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15\", \"7b23de1a-c7ea-4c85-9c25-9375bdee1c71\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.463098362Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.473720789Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15/snapshots/7b23de1a-c7ea-4c85-9c25-9375bdee1c71","method":"GET","req_id":"0526ce05-d5f2-4f1c-9ad8-ff5c4df94724","remote_addr":"[fd00:1122:3344:101::a]:34710","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":32693,"response_code":"200"}
{"msg":"path is \"/data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.473898914Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.483112811Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"snapshot is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15@7b23de1a-c7ea-4c85-9c25-9375bdee1c71","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.492194535Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15\", \"7b23de1a-c7ea-4c85-9c25-9375bdee1c71\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.492224856Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 8c0d83a5-d472-4072-bac6-8e44a67c7b15-7b23de1a-c7ea-4c85-9c25-9375bdee1c71 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.501154906Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"applying SMF actions for region 8c0d83a5-d472-4072-bac6-8e44a67c7b15 running snapshot 7b23de1a-c7ea-4c85-9c25-9375bdee1c71 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.501811814Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.501853635Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15/snapshots/7b23de1a-c7ea-4c85-9c25-9375bdee1c71/run","method":"POST","req_id":"0a10e9cc-f4ee-49c8-a1f8-377ef78e78b6","remote_addr":"[fd00:1122:3344:101::a]:34710","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":27983,"response_code":"200"}
{"msg":"existing directory value /dev/null does not match /data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.507731169Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.507816062Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.507898614Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.507909225Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.507950156Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.508004157Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.5080973Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.508166082Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.508215803Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.508265064Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.516064673Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"creating missing snapshot instance snapshot-8c0d83a5-d472-4072-bac6-8e44a67c7b15-7b23de1a-c7ea-4c85-9c25-9375bdee1c71","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.516551476Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-8c0d83a5-d472-4072-bac6-8e44a67c7b15-7b23de1a-c7ea-4c85-9c25-9375bdee1c71","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.522886003Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-8c0d83a5-d472-4072-bac6-8e44a67c7b15-7b23de1a-c7ea-4c85-9c25-9375bdee1c71","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.541424001Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15/.zfs/snapshot/7b23de1a-c7ea-4c85-9c25-9375bdee1c71","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.54172655Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.541803232Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.541858263Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.541910045Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.541978147Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.549498107Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.556842442Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"region 8c0d83a5-d472-4072-bac6-8e44a67c7b15 running snapshot 7b23de1a-c7ea-4c85-9c25-9375bdee1c71 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:06.556855013Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:08.981166285Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:58594"}
{"msg":"region c81ac63b-d945-4bb9-8b4e-6a263e8758ca state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.040881524Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.041045849Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.04108054Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions","method":"POST","req_id":"8715a1ac-abaa-483d-ab97-46082424695b","remote_addr":"[fd00:1122:3344:101::a]:58594","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":220,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.051182932Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.051228703Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c81ac63b-d945-4bb9-8b4e-6a263e8758ca\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19002, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.106739625Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"region":"c81ac63b-d945-4bb9-8b4e-6a263e8758ca","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.176282739Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"region":"c81ac63b-d945-4bb9-8b4e-6a263e8758ca","component":"worker"}
{"msg":"region c81ac63b-d945-4bb9-8b4e-6a263e8758ca state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.17632847Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.176403372Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.176979369Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177064621Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177144793Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177154943Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177192324Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177241896Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177618126Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177682628Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177734939Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.177781921Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.182166243Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c81ac63b-d945-4bb9-8b4e-6a263e8758ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.182625276Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c81ac63b-d945-4bb9-8b4e-6a263e8758ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.190652401Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.197935914Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c81ac63b-d945-4bb9-8b4e-6a263e8758ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.200426664Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.200704872Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.200782574Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.200826935Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.200872326Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.204249551Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.209910209Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:09.255179614Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions","method":"POST","req_id":"4d94ddfd-9958-4377-ac61-214e281d68f7","remote_addr":"[fd00:1122:3344:101::a]:58594","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.806222837Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63197"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.806296969Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca","method":"GET","req_id":"03a2f124-c02b-4e98-84b7-1b9362698667","remote_addr":"[fd00:1122:3344:101::a]:63197","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"path is \"/data/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.806455843Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.817818521Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"region c81ac63b-d945-4bb9-8b4e-6a263e8758ca state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.825833145Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.82600316Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.826052711Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca","method":"DELETE","req_id":"30356277-f35c-4495-96d8-7ca6e6dfc7f4","remote_addr":"[fd00:1122:3344:101::a]:63197","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19631,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.826691009Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca","method":"GET","req_id":"d45c6668-f52b-4313-8d16-cdc5db0dde51","remote_addr":"[fd00:1122:3344:101::a]:63197","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":524,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-c81ac63b-d945-4bb9-8b4e-6a263e8758ca (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.82710538Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"existing directory value /dev/null does not match /data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.831514484Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.831610036Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.831708769Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.83172248Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.831780821Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.831833283Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8c0d83a5-d472-4072-bac6-8e44a67c7b15","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.832213103Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.832295905Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.832348537Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.832408589Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.836106892Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.837536942Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.847653225Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"region":"c81ac63b-d945-4bb9-8b4e-6a263e8758ca","component":"worker"}
{"msg":"region c81ac63b-d945-4bb9-8b4e-6a263e8758ca state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.896964993Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-06T17:12:40.947120025Z","hostname":"oxz_crucible_c83296b4-dec2-4c81-85c9-f295fcc69098","pid":22950,"uri":"/crucible/0/regions/c81ac63b-d945-4bb9-8b4e-6a263e8758ca","method":"GET","req_id":"d64aa626-1633-4b44-a90d-acd322804e54","remote_addr":"[fd00:1122:3344:101::a]:63197","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":71,"response_code":"200"}