[ Mar 17 16:40:55 Enabled. ]
[ Mar 17 16:40:55 Rereading configuration. ]
[ Mar 17 16:40:57 Rereading configuration. ]
[ Mar 17 16:41:00 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_92457833-9592-4b0e-9272-2eb32511e544/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_92457833-9592-4b0e-9272-2eb32511e544/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.659955762Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.662904645Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.662913245Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.680193176Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:00.758634496Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.023921065Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:47075"}
{"msg":"region 0786c943-08ba-4351-992d-89f6081a2714 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.063764068Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.069061472Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"uri":"/crucible/0/regions","method":"POST","req_id":"fc1cec0e-288b-4fc2-beae-b60218048bbd","remote_addr":"[fd00:1122:3344:101::c]:47075","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":19903,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.069724956Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_92457833-9592-4b0e-9272-2eb32511e544/crucible/regions/0786c943-08ba-4351-992d-89f6081a2714","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.079501205Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_92457833-9592-4b0e-9272-2eb32511e544/crucible/regions/0786c943-08ba-4351-992d-89f6081a2714","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.079536376Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0786c943-08ba-4351-992d-89f6081a2714\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0786c943-08ba-4351-992d-89f6081a2714\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.147581773Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"region":"0786c943-08ba-4351-992d-89f6081a2714","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0786c943-08ba-4351-992d-89f6081a2714\" \"--data\" \"/data/regions/0786c943-08ba-4351-992d-89f6081a2714\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.149599547Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"region":"0786c943-08ba-4351-992d-89f6081a2714","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.278398635Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"region":"0786c943-08ba-4351-992d-89f6081a2714","component":"worker"}
{"msg":"region 0786c943-08ba-4351-992d-89f6081a2714 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.278463326Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.278526528Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0786c943-08ba-4351-992d-89f6081a2714","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.278938506Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0786c943-08ba-4351-992d-89f6081a2714","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.284508356Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.296902501Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0786c943-08ba-4351-992d-89f6081a2714","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.300642421Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0786c943-08ba-4351-992d-89f6081a2714","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.301396018Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.301470789Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.30151422Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.301552691Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.314333695Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.321873186Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"uri":"/crucible/0/regions","method":"POST","req_id":"9be73640-b1a5-4828-9a3f-81b43c0585c4","remote_addr":"[fd00:1122:3344:101::c]:47075","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:56.327757342Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.190435497Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"uri":"/crucible/0/regions/0786c943-08ba-4351-992d-89f6081a2714","method":"GET","req_id":"1aa516d6-7ed7-403f-923b-e9f804dfb1a2","remote_addr":"[fd00:1122:3344:101::c]:47075","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":2073,"response_code":"200"}
{"msg":"path is \"/data/regions/0786c943-08ba-4351-992d-89f6081a2714\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.193144575Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"datafile"}
{"msg":"dataset is oxp_92457833-9592-4b0e-9272-2eb32511e544/crucible/regions/0786c943-08ba-4351-992d-89f6081a2714","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.205639713Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"datafile"}
{"msg":"region 0786c943-08ba-4351-992d-89f6081a2714 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.215828191Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.216004735Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.216060576Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"uri":"/crucible/0/regions/0786c943-08ba-4351-992d-89f6081a2714","method":"DELETE","req_id":"b1055b6f-1ed4-4f70-9824-0328745b2c70","remote_addr":"[fd00:1122:3344:101::c]:47075","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22933,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-0786c943-08ba-4351-992d-89f6081a2714 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.217077088Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.219794106Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"uri":"/crucible/0/regions/0786c943-08ba-4351-992d-89f6081a2714","method":"GET","req_id":"01c2426d-433c-4b65-9ffd-cd08cd505ad2","remote_addr":"[fd00:1122:3344:101::c]:47075","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":36,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.221914981Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_92457833-9592-4b0e-9272-2eb32511e544/crucible/regions/0786c943-08ba-4351-992d-89f6081a2714\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.233551411Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"region":"0786c943-08ba-4351-992d-89f6081a2714","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.286705309Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"uri":"/crucible/0/regions/0786c943-08ba-4351-992d-89f6081a2714","method":"GET","req_id":"a485193e-2518-4f9f-9e86-c8af4a1b21d0","remote_addr":"[fd00:1122:3344:101::c]:47075","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region 0786c943-08ba-4351-992d-89f6081a2714 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.309541618Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-17T16:41:57.450095838Z","hostname":"oxz_crucible_7e64105f-7052-489b-907e-f273bbee772c","pid":22358,"uri":"/crucible/0/regions/0786c943-08ba-4351-992d-89f6081a2714","method":"GET","req_id":"74186545-b5a1-4f2e-8a8f-d0182e3651bd","remote_addr":"[fd00:1122:3344:101::c]:47075","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":44,"response_code":"200"}