[ Oct  3 04:37:55 Enabled. ]
[ Oct  3 04:37:55 Rereading configuration. ]
[ Oct  3 04:38:00 Rereading configuration. ]
[ Oct  3 04:38:03 Executing start method ("/opt/oxide/lib/svc/manifest/crucible/agent.sh"). ]
+ . /lib/svc/share/smf_include.sh
++ SMF_EXIT_OK=0
++ SMF_EXIT_NODAEMON=94
++ SMF_EXIT_ERR_FATAL=95
++ SMF_EXIT_ERR_CONFIG=96
++ SMF_EXIT_MON_DEGRADE=97
++ SMF_EXIT_MON_OFFLINE=98
++ SMF_EXIT_ERR_NOSMF=99
++ SMF_EXIT_ERR_PERM=100
++ svcprop -c -p config/datalink svc:/oxide/crucible/agent:default
+ DATALINK=oxControlService22
++ svcprop -c -p config/gateway svc:/oxide/crucible/agent:default
+ GATEWAY=fd00:1122:3344:101::1
++ svcprop -c -p config/dataset svc:/oxide/crucible/agent:default
+ DATASET=oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible
++ svcprop -c -p config/listen_addr svc:/oxide/crucible/agent:default
+ LISTEN_ADDR=fd00:1122:3344:101::1a
++ svcprop -c -p config/listen_port svc:/oxide/crucible/agent:default
+ LISTEN_PORT=32345
++ svcprop -c -p config/portbase svc:/oxide/crucible/agent:default
+ PORTBASE=19000
++ svcprop -c -p config/downstairs_prefix svc:/oxide/crucible/agent:default
+ DOWNSTAIRS_PREFIX=downstairs
++ svcprop -c -p config/snapshot_prefix svc:/oxide/crucible/agent:default
+ SNAPSHOT_PREFIX=snapshot
+ [[ oxControlService22 == unknown ]]
+ [[ fd00:1122:3344:101::1 == unknown ]]
+ ipadm delete-if oxControlService22
ipadm: Could not delete oxControlService22: Interface does not exist
+ true
+ ipadm create-if -t oxControlService22
+ ipadm set-ifprop -t -p mtu=9000 -m ipv4 oxControlService22
+ ipadm set-ifprop -t -p mtu=9000 -m ipv6 oxControlService22
+ ipadm show-addr oxControlService22/ll
ipadm: Address object not found
+ ipadm create-addr -t -T addrconf oxControlService22/ll
+ ipadm show-addr oxControlService22/omicron6
ipadm: Address object not found
+ ipadm create-addr -t -T static -a fd00:1122:3344:101::1a oxControlService22/omicron6
+ route get -inet6 default -inet6 fd00:1122:3344:101::1
default: not in table
+ route add -inet6 default -inet6 fd00:1122:3344:101::1
add net default: gateway fd00:1122:3344:101::1
+ args=('-D' '/opt/oxide/crucible/bin/crucible-downstairs' '--dataset' "$DATASET" '-l' "[$LISTEN_ADDR]:$LISTEN_PORT" '-P' "$PORTBASE" '-p' "$DOWNSTAIRS_PREFIX" '-s' "$SNAPSHOT_PREFIX")
+ exec /opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible -l '[fd00:1122:3344:101::1a]:32345' -P 19000 -p downstairs -s snapshot
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:38:04.482223728Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:38:04.486034787Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:38:04.486044647Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:38:04.671783702Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.492018218Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:55945"}
{"msg":"region e735b68f-b611-4ac4-932b-59c2289df996 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.556887439Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.562585042Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.564824833Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions","method":"POST","req_id":"9e55d2b6-c797-4ef2-a589-4ca4575fd1cc","remote_addr":"[fd00:1122:3344:101::c]:55945","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":6413,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.572274107Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.572289168Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e735b68f-b611-4ac4-932b-59c2289df996\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/e735b68f-b611-4ac4-932b-59c2289df996\"","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.624167445Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"region":"e735b68f-b611-4ac4-932b-59c2289df996","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.796281233Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions","method":"POST","req_id":"7c47192d-7208-43b1-8c9e-da1d44fa7381","remote_addr":"[fd00:1122:3344:101::c]:55945","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.804830417Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"region":"e735b68f-b611-4ac4-932b-59c2289df996","component":"worker"}
{"msg":"region e735b68f-b611-4ac4-932b-59c2289df996 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.804850238Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.80497825Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.81607411Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.819967921Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.830121964Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.833989024Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.834657066Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.834759978Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.834806339Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.834845739Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.838980324Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:22.84482717Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.309560262Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions","method":"POST","req_id":"f0c9f981-34f2-481a-94db-88231a4aa65a","remote_addr":"[fd00:1122:3344:101::c]:55945","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":104,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.64055159Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:50109"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.640867506Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions/e735b68f-b611-4ac4-932b-59c2289df996","method":"GET","req_id":"4e389af9-f1e9-4326-9be2-9ad9ec503ac9","remote_addr":"[fd00:1122:3344:101::c]:50109","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":206,"response_code":"200"}
{"msg":"path is \"/data/regions/e735b68f-b611-4ac4-932b-59c2289df996\"","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.64111347Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.652098059Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"region e735b68f-b611-4ac4-932b-59c2289df996 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.661388177Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.66155472Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.66159832Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions/e735b68f-b611-4ac4-932b-59c2289df996","method":"DELETE","req_id":"d2e04772-8ea9-4473-beed-9d46f2f59b4d","remote_addr":"[fd00:1122:3344:101::c]:50109","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20512,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.664282799Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions/e735b68f-b611-4ac4-932b-59c2289df996","method":"GET","req_id":"565e3300-3bbb-4f07-8969-1b96499e4e56","remote_addr":"[fd00:1122:3344:101::c]:50109","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":422,"response_code":"200"}
{"msg":"remove downstairs instance: downstairs-e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":50,"time":"2023-10-03T04:39:23.671757164Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"instance states: (Some(Online), None)","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.672446476Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.677493417Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/e735b68f-b611-4ac4-932b-59c2289df996\"","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.688942804Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"region":"e735b68f-b611-4ac4-932b-59c2289df996","component":"worker"}
{"msg":"region e735b68f-b611-4ac4-932b-59c2289df996 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.746018805Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:23.75404508Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions/e735b68f-b611-4ac4-932b-59c2289df996","method":"GET","req_id":"52a50616-287c-4f3a-b42a-7102608fcffd","remote_addr":"[fd00:1122:3344:101::c]:50109","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.03755814Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:57399"}
{"msg":"region b5fa758a-6915-453f-8128-8814d99e7574 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.06854603Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.068730083Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions","method":"POST","req_id":"63f3e67f-14e3-421e-851e-b23c62167cc0","remote_addr":"[fd00:1122:3344:101::c]:57399","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":231,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.068765163Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b5fa758a-6915-453f-8128-8814d99e7574","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.077047513Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b5fa758a-6915-453f-8128-8814d99e7574","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.077073793Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b5fa758a-6915-453f-8128-8814d99e7574\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/b5fa758a-6915-453f-8128-8814d99e7574\"","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.128215977Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"region":"b5fa758a-6915-453f-8128-8814d99e7574","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.185155416Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"region":"b5fa758a-6915-453f-8128-8814d99e7574","component":"worker"}
{"msg":"region b5fa758a-6915-453f-8128-8814d99e7574 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.185215977Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.185286298Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"remove downstairs instance: downstairs-e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":50,"time":"2023-10-03T04:39:26.194622067Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"instance states: (Some(Disabled), None)","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.195252608Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b5fa758a-6915-453f-8128-8814d99e7574","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.195656835Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b5fa758a-6915-453f-8128-8814d99e7574","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.198970575Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.206001392Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b5fa758a-6915-453f-8128-8814d99e7574","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.209384983Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b5fa758a-6915-453f-8128-8814d99e7574","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.210000384Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.210044545Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.210077455Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.210086506Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.232711964Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.24355253Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:39:26.315486719Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions","method":"POST","req_id":"80531635-da54-4287-b6b8-0a2e1a42d749","remote_addr":"[fd00:1122:3344:101::c]:57399","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.195612709Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:43242"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.195905104Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions/b5fa758a-6915-453f-8128-8814d99e7574","method":"GET","req_id":"48ff3e01-edd2-4df2-ba2c-27114acc3777","remote_addr":"[fd00:1122:3344:101::c]:43242","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"path is \"/data/regions/b5fa758a-6915-453f-8128-8814d99e7574\"","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.196034166Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b5fa758a-6915-453f-8128-8814d99e7574","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.207207708Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"region b5fa758a-6915-453f-8128-8814d99e7574 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.21672552Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.216927324Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.216978235Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions/b5fa758a-6915-453f-8128-8814d99e7574","method":"DELETE","req_id":"2ed32f00-d08c-4460-9738-dbf1a8e176b3","remote_addr":"[fd00:1122:3344:101::c]:43242","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20969,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.219858877Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions/b5fa758a-6915-453f-8128-8814d99e7574","method":"GET","req_id":"4abbce5f-9f1d-45d1-bbc9-a64a480b8354","remote_addr":"[fd00:1122:3344:101::c]:43242","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":470,"response_code":"200"}
{"msg":"remove downstairs instance: downstairs-e735b68f-b611-4ac4-932b-59c2289df996","v":0,"name":"crucible-agent","level":50,"time":"2023-10-03T04:42:19.226905824Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"instance states: (Some(Disabled), None)","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.227107008Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"remove downstairs instance: downstairs-b5fa758a-6915-453f-8128-8814d99e7574","v":0,"name":"crucible-agent","level":50,"time":"2023-10-03T04:42:19.227458924Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"instance states: (Some(Online), None)","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.228093816Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.237566156Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/b5fa758a-6915-453f-8128-8814d99e7574\"","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.249699366Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"region":"b5fa758a-6915-453f-8128-8814d99e7574","component":"worker"}
{"msg":"region b5fa758a-6915-453f-8128-8814d99e7574 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.303567868Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2023-10-03T04:42:19.337791347Z","hostname":"oxz_crucible_e0f9ddf5-627e-4c61-b74b-671314e193bf","pid":21692,"uri":"/crucible/0/regions/b5fa758a-6915-453f-8128-8814d99e7574","method":"GET","req_id":"9c11ba2d-8a2b-4ab0-bbca-da40c8825192","remote_addr":"[fd00:1122:3344:101::c]:43242","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":73,"response_code":"200"}