[ Jan 23 18:48:08 Enabled. ]
[ Jan 23 18:48:08 Rereading configuration. ]
[ Jan 23 18:48:09 Rereading configuration. ]
[ Jan 23 18:48:10 Executing start method ("/opt/oxide/lib/svc/manifest/crucible/agent.sh"). ]
+ . /lib/svc/share/smf_include.sh
++ SMF_EXIT_OK=0
++ SMF_EXIT_NODAEMON=94
++ SMF_EXIT_ERR_FATAL=95
++ SMF_EXIT_ERR_CONFIG=96
++ SMF_EXIT_MON_DEGRADE=97
++ SMF_EXIT_MON_OFFLINE=98
++ SMF_EXIT_ERR_NOSMF=99
++ SMF_EXIT_ERR_PERM=100
++ svcprop -c -p config/datalink svc:/oxide/crucible/agent:default
+ DATALINK=oxControlService22
++ svcprop -c -p config/gateway svc:/oxide/crucible/agent:default
+ GATEWAY=fd00:1122:3344:101::1
++ svcprop -c -p config/dataset svc:/oxide/crucible/agent:default
+ DATASET=oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible
++ svcprop -c -p config/listen_addr svc:/oxide/crucible/agent:default
+ LISTEN_ADDR=fd00:1122:3344:101::18
++ svcprop -c -p config/listen_port svc:/oxide/crucible/agent:default
+ LISTEN_PORT=32345
++ svcprop -c -p config/portbase svc:/oxide/crucible/agent:default
+ PORTBASE=19000
++ svcprop -c -p config/downstairs_prefix svc:/oxide/crucible/agent:default
+ DOWNSTAIRS_PREFIX=downstairs
++ svcprop -c -p config/snapshot_prefix svc:/oxide/crucible/agent:default
+ SNAPSHOT_PREFIX=snapshot
+ [[ oxControlService22 == unknown ]]
+ [[ fd00:1122:3344:101::1 == unknown ]]
+ ipadm delete-if oxControlService22
ipadm: Could not delete oxControlService22: Interface does not exist
+ true
+ ipadm create-if -t oxControlService22
+ ipadm set-ifprop -t -p mtu=9000 -m ipv4 oxControlService22
+ ipadm set-ifprop -t -p mtu=9000 -m ipv6 oxControlService22
+ ipadm show-addr oxControlService22/ll
ipadm: Address object not found
+ ipadm create-addr -t -T addrconf oxControlService22/ll
+ ipadm show-addr oxControlService22/omicron6
ipadm: Address object not found
+ ipadm create-addr -t -T static -a fd00:1122:3344:101::18 oxControlService22/omicron6
+ route get -inet6 default -inet6 fd00:1122:3344:101::1
default: not in table
+ route add -inet6 default -inet6 fd00:1122:3344:101::1
add net default: gateway fd00:1122:3344:101::1
+ args=('-D' '/opt/oxide/crucible/bin/crucible-downstairs' '--dataset' "$DATASET" '-l' "[$LISTEN_ADDR]:$LISTEN_PORT" '-P' "$PORTBASE" '-p' "$DOWNSTAIRS_PREFIX" '-s' "$SNAPSHOT_PREFIX")
+ exec /opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible -l '[fd00:1122:3344:101::18]:32345' -P 19000 -p downstairs -s snapshot
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:11.567105099Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:11.579005189Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:11.579017689Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:11.766479002Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.456811408Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46924"}
{"msg":"region 10a561e2-8496-4148-95d5-c8eaa6143416 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.522997079Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.527286903Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.527311464Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions","method":"POST","req_id":"1673a3a5-237d-4e4d-9323-036653c2dced","remote_addr":"[fd00:1122:3344:101::a]:46924","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":10855,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/10a561e2-8496-4148-95d5-c8eaa6143416","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.537011069Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/10a561e2-8496-4148-95d5-c8eaa6143416","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.537061381Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"10a561e2-8496-4148-95d5-c8eaa6143416\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/10a561e2-8496-4148-95d5-c8eaa6143416\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.58880589Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"region":"10a561e2-8496-4148-95d5-c8eaa6143416","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.833314121Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions","method":"POST","req_id":"db7dfd16-477a-4302-ac82-bcde3bf98aab","remote_addr":"[fd00:1122:3344:101::a]:46924","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.905922608Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"region":"10a561e2-8496-4148-95d5-c8eaa6143416","component":"worker"}
{"msg":"region 10a561e2-8496-4148-95d5-c8eaa6143416 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.905958789Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.906098313Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-10a561e2-8496-4148-95d5-c8eaa6143416","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.906561984Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-10a561e2-8496-4148-95d5-c8eaa6143416","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.910186652Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.923998739Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-10a561e2-8496-4148-95d5-c8eaa6143416","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.927471023Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/10a561e2-8496-4148-95d5-c8eaa6143416","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.928206831Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.928340094Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.928386085Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.928426416Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.933753436Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:53.938920752Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:54.562769236Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions","method":"POST","req_id":"3a78d989-2741-4ac7-ba34-241007fa1c6e","remote_addr":"[fd00:1122:3344:101::a]:46924","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":95,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:54.982012379Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54267"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:54.982140142Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions/10a561e2-8496-4148-95d5-c8eaa6143416","method":"GET","req_id":"e88b5493-aedb-4db1-8707-00bfdd89f1b0","remote_addr":"[fd00:1122:3344:101::a]:54267","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":97,"response_code":"200"}
{"msg":"path is \"/data/regions/10a561e2-8496-4148-95d5-c8eaa6143416\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:54.982365048Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/10a561e2-8496-4148-95d5-c8eaa6143416","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:54.992027153Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"region 10a561e2-8496-4148-95d5-c8eaa6143416 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.001321179Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.001457042Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.001511704Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions/10a561e2-8496-4148-95d5-c8eaa6143416","method":"DELETE","req_id":"34d402b3-4388-4713-8a38-f1d35ff11b4b","remote_addr":"[fd00:1122:3344:101::a]:54267","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19187,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.001663617Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions/10a561e2-8496-4148-95d5-c8eaa6143416","method":"GET","req_id":"8d1a0300-6682-4c34-98b6-15a246b437ab","remote_addr":"[fd00:1122:3344:101::a]:54267","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-10a561e2-8496-4148-95d5-c8eaa6143416 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.002472947Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.00670863Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/10a561e2-8496-4148-95d5-c8eaa6143416\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.017358659Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"region":"10a561e2-8496-4148-95d5-c8eaa6143416","component":"worker"}
{"msg":"region 10a561e2-8496-4148-95d5-c8eaa6143416 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.067710865Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:55.109088152Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions/10a561e2-8496-4148-95d5-c8eaa6143416","method":"GET","req_id":"8423fbee-4372-4217-bf62-882685b8ce1e","remote_addr":"[fd00:1122:3344:101::a]:54267","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":94,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.358777068Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:38691"}
{"msg":"region af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.419323172Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.419584588Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.419604749Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions","method":"POST","req_id":"775e8fcf-6483-4d61-9b3d-d8a182056382","remote_addr":"[fd00:1122:3344:101::a]:38691","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":334,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.430468333Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.430496164Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.477504488Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"region":"af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.552969945Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"region":"af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","component":"worker"}
{"msg":"region af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.553015026Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.553112609Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.554494642Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.558000937Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.56756734Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.57041941Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions","method":"POST","req_id":"8844c342-b497-4ad9-be78-a3ecd6215f06","remote_addr":"[fd00:1122:3344:101::a]:38691","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.570949753Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.57124723Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.571302451Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.571347672Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.571392063Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.575743569Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:48:56.581138091Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.223972166Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:42473"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.224214152Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","method":"GET","req_id":"650495df-5bc4-4ed1-89e5-05ffb288686f","remote_addr":"[fd00:1122:3344:101::a]:42473","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"path is \"/data/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.224251533Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.23396666Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a@e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.243194724Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a\", \"e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.243235175Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.252646004Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a/snapshots/e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe","method":"GET","req_id":"770b607d-e1ab-47c7-a750-53bf2bbf328a","remote_addr":"[fd00:1122:3344:101::a]:42473","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":28458,"response_code":"200"}
{"msg":"path is \"/data/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.252774667Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"dataset is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.261661224Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"snapshot is oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a@e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.270197831Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crucible/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a\", \"e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.270226942Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a-e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.279267032Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}
{"msg":"applying SMF actions for region af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a running snapshot e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.279826426Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.279864657Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"uri":"/crucible/0/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a/snapshots/e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe/run","method":"POST","req_id":"f1cf3051-ccf0-44c4-aa09-17b3becee851","remote_addr":"[fd00:1122:3344:101::a]:42473","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":27123,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a-e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.288586939Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a-e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.29273381Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a-e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.308210347Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a/.zfs/snapshot/e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.308492694Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.308548675Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.308592686Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.308633457Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.308683708Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.31287688Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.317901972Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"worker"}
{"msg":"region af8e6c1a-ed98-4c15-8eb2-3e2bebec0f5a running snapshot e4f2903b-6fa7-434c-aa8b-fd694ad2f3fe state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-23T18:54:38.317914933Z","hostname":"oxz_crucible_3dfef7f9-5740-4aed-9fdb-dcf7e7cc60c5","pid":24946,"component":"datafile"}