[ Feb  1 23:04:39 Enabled. ]
[ Feb  1 23:04:39 Rereading configuration. ]
[ Feb  1 23:04:41 Rereading configuration. ]
[ Feb  1 23:04:45 Executing start method ("/opt/oxide/lib/svc/manifest/crucible/agent.sh"). ]
+ . /lib/svc/share/smf_include.sh
++ SMF_EXIT_OK=0
++ SMF_EXIT_NODAEMON=94
++ SMF_EXIT_ERR_FATAL=95
++ SMF_EXIT_ERR_CONFIG=96
++ SMF_EXIT_MON_DEGRADE=97
++ SMF_EXIT_MON_OFFLINE=98
++ SMF_EXIT_ERR_NOSMF=99
++ SMF_EXIT_ERR_PERM=100
++ svcprop -c -p config/datalink svc:/oxide/crucible/agent:default
+ DATALINK=oxControlService18
++ svcprop -c -p config/gateway svc:/oxide/crucible/agent:default
+ GATEWAY=fd00:1122:3344:101::1
++ svcprop -c -p config/dataset svc:/oxide/crucible/agent:default
+ DATASET=oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible
++ svcprop -c -p config/listen_addr svc:/oxide/crucible/agent:default
+ LISTEN_ADDR=fd00:1122:3344:101::13
++ svcprop -c -p config/listen_port svc:/oxide/crucible/agent:default
+ LISTEN_PORT=32345
++ svcprop -c -p config/portbase svc:/oxide/crucible/agent:default
+ PORTBASE=19000
++ svcprop -c -p config/downstairs_prefix svc:/oxide/crucible/agent:default
+ DOWNSTAIRS_PREFIX=downstairs
++ svcprop -c -p config/snapshot_prefix svc:/oxide/crucible/agent:default
+ SNAPSHOT_PREFIX=snapshot
+ [[ oxControlService18 == unknown ]]
+ [[ fd00:1122:3344:101::1 == unknown ]]
+ ipadm delete-if oxControlService18
ipadm: Could not delete oxControlService18: Interface does not exist
+ true
+ ipadm create-if -t oxControlService18
+ ipadm set-ifprop -t -p mtu=9000 -m ipv4 oxControlService18
+ ipadm set-ifprop -t -p mtu=9000 -m ipv6 oxControlService18
+ ipadm show-addr oxControlService18/ll
ipadm: Address object not found
+ ipadm create-addr -t -T addrconf oxControlService18/ll
+ ipadm show-addr oxControlService18/omicron6
ipadm: Address object not found
+ ipadm create-addr -t -T static -a fd00:1122:3344:101::13 oxControlService18/omicron6
+ route get -inet6 default -inet6 fd00:1122:3344:101::1
default: not in table
+ route add -inet6 default -inet6 fd00:1122:3344:101::1
add net default: gateway fd00:1122:3344:101::1
+ args=('-D' '/opt/oxide/crucible/bin/crucible-downstairs' '--dataset' "$DATASET" '-l' "[$LISTEN_ADDR]:$LISTEN_PORT" '-P' "$PORTBASE" '-p' "$DOWNSTAIRS_PREFIX" '-s' "$SNAPSHOT_PREFIX")
+ exec /opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l '[fd00:1122:3344:101::13]:32345' -P 19000 -p downstairs -s snapshot
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:04:45.880908908Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:04:45.885616527Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:04:45.885628267Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:04:46.11537488Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:36.890285993Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:63261"}
{"msg":"region bed36ab7-b152-4c54-b074-edb681429b6a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:36.89186813Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:36.892077815Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"211f1295-5bbf-45d4-9df4-222cce0cd3cc","remote_addr":"[fd00:1122:3344:101::c]:63261","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":410,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:36.89617391Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/bed36ab7-b152-4c54-b074-edb681429b6a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:36.905814604Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/bed36ab7-b152-4c54-b074-edb681429b6a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:36.905846025Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"bed36ab7-b152-4c54-b074-edb681429b6a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/bed36ab7-b152-4c54-b074-edb681429b6a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:36.969226139Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"region":"bed36ab7-b152-4c54-b074-edb681429b6a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.079403661Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"ab05c504-a719-47a0-8834-ac32efbc1f8e","remote_addr":"[fd00:1122:3344:101::c]:63261","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":672,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.240455495Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"region":"bed36ab7-b152-4c54-b074-edb681429b6a","component":"worker"}
{"msg":"region bed36ab7-b152-4c54-b074-edb681429b6a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.240502047Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.24062221Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-bed36ab7-b152-4c54-b074-edb681429b6a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.24107135Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bed36ab7-b152-4c54-b074-edb681429b6a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.244927759Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.253833767Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bed36ab7-b152-4c54-b074-edb681429b6a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.25699224Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bed36ab7-b152-4c54-b074-edb681429b6a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.257755988Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.257881991Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.257926082Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.257965603Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.263333327Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.268972029Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.559197097Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"17dd32f4-58bc-4b10-8ecd-27c3066833e7","remote_addr":"[fd00:1122:3344:101::c]:63261","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.892418536Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37934"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.892563009Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions/bed36ab7-b152-4c54-b074-edb681429b6a","method":"GET","req_id":"e6329625-3476-4d41-aa80-9e3250cbb7b3","remote_addr":"[fd00:1122:3344:101::c]:37934","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":101,"response_code":"200"}
{"msg":"path is \"/data/regions/bed36ab7-b152-4c54-b074-edb681429b6a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.892857126Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/bed36ab7-b152-4c54-b074-edb681429b6a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.901981038Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"region bed36ab7-b152-4c54-b074-edb681429b6a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.911039059Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.911188812Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.911229013Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions/bed36ab7-b152-4c54-b074-edb681429b6a","method":"DELETE","req_id":"9f39cc91-46d4-4025-a668-ced2fb4ef3ab","remote_addr":"[fd00:1122:3344:101::c]:37934","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":18438,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.912030512Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions/bed36ab7-b152-4c54-b074-edb681429b6a","method":"GET","req_id":"bf89acfc-d2be-4b99-b938-d6098efd5541","remote_addr":"[fd00:1122:3344:101::c]:37934","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":541,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-bed36ab7-b152-4c54-b074-edb681429b6a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.912430921Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.916602138Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/bed36ab7-b152-4c54-b074-edb681429b6a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.92614438Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"region":"bed36ab7-b152-4c54-b074-edb681429b6a","component":"worker"}
{"msg":"region bed36ab7-b152-4c54-b074-edb681429b6a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.977030183Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:05:37.994410307Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions/bed36ab7-b152-4c54-b074-edb681429b6a","method":"GET","req_id":"da0a4d86-484c-4441-ac8b-d20b47c403c6","remote_addr":"[fd00:1122:3344:101::c]:37934","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.10297469Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:32976"}
{"msg":"region 68270274-3c15-4bad-a0b3-d14e647daea3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.134951773Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.13524043Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.135272041Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"adb84e33-3d3c-44a5-9024-311826a86608","remote_addr":"[fd00:1122:3344:101::a]:32976","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":343,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/68270274-3c15-4bad-a0b3-d14e647daea3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.143654046Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/68270274-3c15-4bad-a0b3-d14e647daea3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.143700287Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"68270274-3c15-4bad-a0b3-d14e647daea3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/68270274-3c15-4bad-a0b3-d14e647daea3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.201346617Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"region":"68270274-3c15-4bad-a0b3-d14e647daea3","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.277835826Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"region":"68270274-3c15-4bad-a0b3-d14e647daea3","component":"worker"}
{"msg":"region 68270274-3c15-4bad-a0b3-d14e647daea3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.277884537Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.277971149Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-68270274-3c15-4bad-a0b3-d14e647daea3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.279332711Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-68270274-3c15-4bad-a0b3-d14e647daea3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.28316254Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.291078364Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-68270274-3c15-4bad-a0b3-d14e647daea3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.294730049Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/68270274-3c15-4bad-a0b3-d14e647daea3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.295006105Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.295061222Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.295100486Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.295139964Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.299044949Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.303874571Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:06:20.311607521Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"095f7baf-6511-4189-aa1b-a941506cde9a","remote_addr":"[fd00:1122:3344:101::a]:32976","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.644763026Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:43695"}
{"msg":"region 28bc1d52-ccb4-4fcb-8755-c6b481e3ab11 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.64491971Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.6453533Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.64536257Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"6bc927f8-eebc-49bb-9e7a-4c0f1c58ae8f","remote_addr":"[fd00:1122:3344:101::a]:43695","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":513,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/28bc1d52-ccb4-4fcb-8755-c6b481e3ab11","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.655397053Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/28bc1d52-ccb4-4fcb-8755-c6b481e3ab11","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.655424754Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"28bc1d52-ccb4-4fcb-8755-c6b481e3ab11\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/28bc1d52-ccb4-4fcb-8755-c6b481e3ab11\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.723095288Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"region":"28bc1d52-ccb4-4fcb-8755-c6b481e3ab11","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.795638184Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"region":"28bc1d52-ccb4-4fcb-8755-c6b481e3ab11","component":"worker"}
{"msg":"region 28bc1d52-ccb4-4fcb-8755-c6b481e3ab11 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.795694786Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.795811648Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-28bc1d52-ccb4-4fcb-8755-c6b481e3ab11","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.796857443Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-28bc1d52-ccb4-4fcb-8755-c6b481e3ab11","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.802455023Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.816562631Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-28bc1d52-ccb4-4fcb-8755-c6b481e3ab11","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.820700277Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/28bc1d52-ccb4-4fcb-8755-c6b481e3ab11","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.820976584Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.821040175Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.821084396Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.821125647Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.824650769Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.830515746Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:45.970326006Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions","method":"POST","req_id":"e94e5987-986c-49ad-b82f-d87675b490b2","remote_addr":"[fd00:1122:3344:101::a]:43695","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:46.983317072Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60254"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:46.983393894Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions/68270274-3c15-4bad-a0b3-d14e647daea3","method":"GET","req_id":"0c0f962f-067a-49ac-b441-6c941ab3ffcb","remote_addr":"[fd00:1122:3344:101::a]:60254","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"path is \"/data/regions/68270274-3c15-4bad-a0b3-d14e647daea3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:46.987144361Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/68270274-3c15-4bad-a0b3-d14e647daea3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:46.996399826Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"snapshot is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/68270274-3c15-4bad-a0b3-d14e647daea3@02bf3710-1c87-41ec-aee5-3f0193287fee","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.00475251Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/68270274-3c15-4bad-a0b3-d14e647daea3\", \"02bf3710-1c87-41ec-aee5-3f0193287fee\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.004789211Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.016138725Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions/68270274-3c15-4bad-a0b3-d14e647daea3/snapshots/02bf3710-1c87-41ec-aee5-3f0193287fee","method":"GET","req_id":"98aab038-b1fb-488e-b92b-634fb6e62bbd","remote_addr":"[fd00:1122:3344:101::a]:60254","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":32558,"response_code":"200"}
{"msg":"path is \"/data/regions/68270274-3c15-4bad-a0b3-d14e647daea3\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.01635071Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"dataset is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/68270274-3c15-4bad-a0b3-d14e647daea3","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.025442011Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"snapshot is oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/68270274-3c15-4bad-a0b3-d14e647daea3@02bf3710-1c87-41ec-aee5-3f0193287fee","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.033930459Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/68270274-3c15-4bad-a0b3-d14e647daea3\", \"02bf3710-1c87-41ec-aee5-3f0193287fee\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.03396555Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 68270274-3c15-4bad-a0b3-d14e647daea3-02bf3710-1c87-41ec-aee5-3f0193287fee state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.042993719Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}
{"msg":"applying SMF actions for region 68270274-3c15-4bad-a0b3-d14e647daea3 running snapshot 02bf3710-1c87-41ec-aee5-3f0193287fee (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.043152313Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.043205924Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"uri":"/crucible/0/regions/68270274-3c15-4bad-a0b3-d14e647daea3/snapshots/02bf3710-1c87-41ec-aee5-3f0193287fee/run","method":"POST","req_id":"1086f4b5-a3c9-46ae-8c45-64dc16561644","remote_addr":"[fd00:1122:3344:101::a]:60254","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26891,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-68270274-3c15-4bad-a0b3-d14e647daea3-02bf3710-1c87-41ec-aee5-3f0193287fee","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.046166803Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-68270274-3c15-4bad-a0b3-d14e647daea3-02bf3710-1c87-41ec-aee5-3f0193287fee","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.051690252Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-68270274-3c15-4bad-a0b3-d14e647daea3-02bf3710-1c87-41ec-aee5-3f0193287fee","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.061694844Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/68270274-3c15-4bad-a0b3-d14e647daea3/.zfs/snapshot/02bf3710-1c87-41ec-aee5-3f0193287fee","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.062038942Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.062088853Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.062127104Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.062169175Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.062218307Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.065957513Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.070613462Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"worker"}
{"msg":"region 68270274-3c15-4bad-a0b3-d14e647daea3 running snapshot 02bf3710-1c87-41ec-aee5-3f0193287fee state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-01T23:11:47.070631522Z","hostname":"oxz_crucible_6bb658c5-cd67-45df-be53-726b1e038112","pid":24006,"component":"datafile"}