[ Jan 16 00:47:54 Enabled. ]
[ Jan 16 00:47:54 Rereading configuration. ]
[ Jan 16 00:47:56 Rereading configuration. ]
[ Jan 16 00:47:59 Executing start method ("/opt/oxide/lib/svc/manifest/crucible/agent.sh"). ]
+ . /lib/svc/share/smf_include.sh
++ SMF_EXIT_OK=0
++ SMF_EXIT_NODAEMON=94
++ SMF_EXIT_ERR_FATAL=95
++ SMF_EXIT_ERR_CONFIG=96
++ SMF_EXIT_MON_DEGRADE=97
++ SMF_EXIT_MON_OFFLINE=98
++ SMF_EXIT_ERR_NOSMF=99
++ SMF_EXIT_ERR_PERM=100
++ svcprop -c -p config/datalink svc:/oxide/crucible/agent:default
+ DATALINK=oxControlService13
++ svcprop -c -p config/gateway svc:/oxide/crucible/agent:default
+ GATEWAY=fd00:1122:3344:101::1
++ svcprop -c -p config/dataset svc:/oxide/crucible/agent:default
+ DATASET=oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible
++ svcprop -c -p config/listen_addr svc:/oxide/crucible/agent:default
+ LISTEN_ADDR=fd00:1122:3344:101::14
++ svcprop -c -p config/listen_port svc:/oxide/crucible/agent:default
+ LISTEN_PORT=32345
++ svcprop -c -p config/portbase svc:/oxide/crucible/agent:default
+ PORTBASE=19000
++ svcprop -c -p config/downstairs_prefix svc:/oxide/crucible/agent:default
+ DOWNSTAIRS_PREFIX=downstairs
++ svcprop -c -p config/snapshot_prefix svc:/oxide/crucible/agent:default
+ SNAPSHOT_PREFIX=snapshot
+ [[ oxControlService13 == unknown ]]
+ [[ fd00:1122:3344:101::1 == unknown ]]
+ ipadm delete-if oxControlService13
ipadm: Could not delete oxControlService13: Interface does not exist
+ true
+ ipadm create-if -t oxControlService13
+ ipadm set-ifprop -t -p mtu=9000 -m ipv4 oxControlService13
+ ipadm set-ifprop -t -p mtu=9000 -m ipv6 oxControlService13
+ ipadm show-addr oxControlService13/ll
ipadm: Address object not found
+ ipadm create-addr -t -T addrconf oxControlService13/ll
+ ipadm show-addr oxControlService13/omicron6
ipadm: Address object not found
+ ipadm create-addr -t -T static -a fd00:1122:3344:101::14 oxControlService13/omicron6
+ route get -inet6 default -inet6 fd00:1122:3344:101::1
default: not in table
+ route add -inet6 default -inet6 fd00:1122:3344:101::1
add net default: gateway fd00:1122:3344:101::1
+ args=('-D' '/opt/oxide/crucible/bin/crucible-downstairs' '--dataset' "$DATASET" '-l' "[$LISTEN_ADDR]:$LISTEN_PORT" '-P' "$PORTBASE" '-p' "$DOWNSTAIRS_PREFIX" '-s' "$SNAPSHOT_PREFIX")
+ exec /opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible -l '[fd00:1122:3344:101::14]:32345' -P 19000 -p downstairs -s snapshot
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:47:59.983774916Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:47:59.987394603Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:47:59.987407613Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:48:00.439110469Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.223666374Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55943"}
{"msg":"region 291d6d65-dd89-4823-8704-68021a70c574 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.286649098Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.286956561Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"c16970d8-f7ed-4000-bac7-6f1753d6d970","remote_addr":"[fd00:1122:3344:101::a]:55943","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":3722,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.291417806Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/291d6d65-dd89-4823-8704-68021a70c574","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.301485467Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/291d6d65-dd89-4823-8704-68021a70c574","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.301517907Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"291d6d65-dd89-4823-8704-68021a70c574\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/291d6d65-dd89-4823-8704-68021a70c574\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.365585212Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"region":"291d6d65-dd89-4823-8704-68021a70c574","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.543829396Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"20a1ed31-6629-4241-b25d-50ba468ce3c2","remote_addr":"[fd00:1122:3344:101::a]:55943","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":87,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.695852216Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"region":"291d6d65-dd89-4823-8704-68021a70c574","component":"worker"}
{"msg":"region 291d6d65-dd89-4823-8704-68021a70c574 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.695883356Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.696008427Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-291d6d65-dd89-4823-8704-68021a70c574","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.696438441Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-291d6d65-dd89-4823-8704-68021a70c574","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.700103618Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.709804786Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-291d6d65-dd89-4823-8704-68021a70c574","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.713081289Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/291d6d65-dd89-4823-8704-68021a70c574","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.713813427Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.713954328Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.713996698Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.714036179Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.718274961Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:02.722644586Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.249984793Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"c4993a3a-79b3-430e-97be-6cb1f70ce526","remote_addr":"[fd00:1122:3344:101::a]:55943","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.858390835Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:46635"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.858525747Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions/291d6d65-dd89-4823-8704-68021a70c574","method":"GET","req_id":"45f20c0f-b406-4e8d-b48e-84c7bbf92406","remote_addr":"[fd00:1122:3344:101::a]:46635","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":104,"response_code":"200"}
{"msg":"path is \"/data/regions/291d6d65-dd89-4823-8704-68021a70c574\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.858738039Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/291d6d65-dd89-4823-8704-68021a70c574","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.868114403Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"region 291d6d65-dd89-4823-8704-68021a70c574 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.876976472Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.877118534Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.877177604Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions/291d6d65-dd89-4823-8704-68021a70c574","method":"DELETE","req_id":"c8e018f8-8cb0-4f98-a9fd-8c0fe281e1fb","remote_addr":"[fd00:1122:3344:101::a]:46635","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18483,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.877930962Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions/291d6d65-dd89-4823-8704-68021a70c574","method":"GET","req_id":"6b9e3abc-9eb6-4e04-a8ef-6942bd4b128b","remote_addr":"[fd00:1122:3344:101::a]:46635","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":632,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-291d6d65-dd89-4823-8704-68021a70c574 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.878216955Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.882301356Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/291d6d65-dd89-4823-8704-68021a70c574\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.892229216Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"region":"291d6d65-dd89-4823-8704-68021a70c574","component":"worker"}
{"msg":"region 291d6d65-dd89-4823-8704-68021a70c574 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:03.942135018Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:49:04.013252804Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions/291d6d65-dd89-4823-8704-68021a70c574","method":"GET","req_id":"874d901b-b857-40ac-b9de-d36fe0e3fd3b","remote_addr":"[fd00:1122:3344:101::a]:46635","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":51,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:27.893755437Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:50917"}
{"msg":"region 5782779e-8925-425d-88b0-41222753f71e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:27.894413094Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:27.89502906Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:27.89504502Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"cbf8d0f5-4a44-45c5-9762-95421573d997","remote_addr":"[fd00:1122:3344:101::a]:50917","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":1191,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/5782779e-8925-425d-88b0-41222753f71e","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:27.905866939Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/5782779e-8925-425d-88b0-41222753f71e","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:27.905897769Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"5782779e-8925-425d-88b0-41222753f71e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/5782779e-8925-425d-88b0-41222753f71e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:27.98345097Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"region":"5782779e-8925-425d-88b0-41222753f71e","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.198894308Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"5e484a8b-9810-4a12-ae00-bed766de492f","remote_addr":"[fd00:1122:3344:101::a]:50917","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.22496603Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"region":"5782779e-8925-425d-88b0-41222753f71e","component":"worker"}
{"msg":"region 5782779e-8925-425d-88b0-41222753f71e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.224989991Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.225107612Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-5782779e-8925-425d-88b0-41222753f71e","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.226524636Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5782779e-8925-425d-88b0-41222753f71e","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.235404306Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.261617889Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5782779e-8925-425d-88b0-41222753f71e","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.266176625Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5782779e-8925-425d-88b0-41222753f71e","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.266489278Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.266545849Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.26659848Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.26664659Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.273203056Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.279046585Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:28.780045187Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"bd90ff5a-2de1-44a4-b1d7-72a0745ad910","remote_addr":"[fd00:1122:3344:101::a]:50917","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.027465168Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63123"}
{"msg":"region ca79c50a-b7bb-4982-ab50-21cc08f93d5f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.028196076Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.028366687Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.028374287Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"115c916f-ab9c-48c4-bd01-784bd2fda256","remote_addr":"[fd00:1122:3344:101::a]:63123","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":883,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.038207646Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.038240917Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ca79c50a-b7bb-4982-ab50-21cc08f93d5f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.104816727Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"region":"ca79c50a-b7bb-4982-ab50-21cc08f93d5f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.212148937Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"82625b2e-dcef-4e10-8b18-67996cdeb085","remote_addr":"[fd00:1122:3344:101::a]:63123","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":119,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.22935523Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"region":"ca79c50a-b7bb-4982-ab50-21cc08f93d5f","component":"worker"}
{"msg":"region ca79c50a-b7bb-4982-ab50-21cc08f93d5f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.229397451Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.229563372Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ca79c50a-b7bb-4982-ab50-21cc08f93d5f","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.231683564Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ca79c50a-b7bb-4982-ab50-21cc08f93d5f","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.239449382Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.260043769Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ca79c50a-b7bb-4982-ab50-21cc08f93d5f","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.270572985Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.270945779Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.271015129Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.27107885Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.271141191Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.278753627Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.287369484Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:54:32.826154346Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions","method":"POST","req_id":"310a7162-40f6-44b3-b51d-587b914db2b9","remote_addr":"[fd00:1122:3344:101::a]:63123","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.391446251Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:59221"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.391510761Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f","method":"GET","req_id":"75f84782-4b50-4c94-b2dd-9826e0381347","remote_addr":"[fd00:1122:3344:101::a]:59221","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"path is \"/data/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.391704363Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"dataset is oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.402732144Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"region ca79c50a-b7bb-4982-ab50-21cc08f93d5f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.410889186Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.411075138Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.411141629Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f","method":"DELETE","req_id":"ec5ab9ac-ce67-4182-8e8d-14b785a474f6","remote_addr":"[fd00:1122:3344:101::a]:59221","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":19463,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.41129173Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f","method":"GET","req_id":"b23eea1f-64db-4b4b-9880-8e04cda8a90e","remote_addr":"[fd00:1122:3344:101::a]:59221","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-ca79c50a-b7bb-4982-ab50-21cc08f93d5f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.412850686Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.418590374Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crucible/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.427508853Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"region":"ca79c50a-b7bb-4982-ab50-21cc08f93d5f","component":"worker"}
{"msg":"region ca79c50a-b7bb-4982-ab50-21cc08f93d5f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.470459285Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-01-16T00:55:03.501667579Z","hostname":"oxz_crucible_ec64e9d0-0a28-424c-8a11-866f99eec730","pid":25499,"uri":"/crucible/0/regions/ca79c50a-b7bb-4982-ab50-21cc08f93d5f","method":"GET","req_id":"cb99a846-3388-4def-a72c-f8194cdc8767","remote_addr":"[fd00:1122:3344:101::a]:59221","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":89,"response_code":"200"}