[ Mar  8 00:40:14 Enabled. ]
[ Mar  8 00:40:14 Rereading configuration. ]
[ Mar  8 00:40:16 Rereading configuration. ]
[ Mar  8 00:40:21 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f50d55fe-477d-42a7-afdb-d417b796d9e5/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f50d55fe-477d-42a7-afdb-d417b796d9e5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:21.669050893Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:21.67336081Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:21.67337231Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:21.690642426Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:21.795881233Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.56399075Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:64830"}
{"msg":"region ccf922d8-039a-4f7a-b7c5-b0604764c73c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.580762783Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.58182475Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.586749511Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions","method":"POST","req_id":"82b4e0e5-5fc0-47be-98e0-2e5bade2dc15","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":9110,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_f50d55fe-477d-42a7-afdb-d417b796d9e5/crucible/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.598233474Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f50d55fe-477d-42a7-afdb-d417b796d9e5/crucible/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.598287906Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ccf922d8-039a-4f7a-b7c5-b0604764c73c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.713233892Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"region":"ccf922d8-039a-4f7a-b7c5-b0604764c73c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ccf922d8-039a-4f7a-b7c5-b0604764c73c\" \"--data\" \"/data/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.713337995Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"region":"ccf922d8-039a-4f7a-b7c5-b0604764c73c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.823087093Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions","method":"POST","req_id":"3064adb2-faa7-4a2d-9672-50f4233cc4b6","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.876444759Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"region":"ccf922d8-039a-4f7a-b7c5-b0604764c73c","component":"worker"}
{"msg":"region ccf922d8-039a-4f7a-b7c5-b0604764c73c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.87686052Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.877978057Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ccf922d8-039a-4f7a-b7c5-b0604764c73c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.87850312Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ccf922d8-039a-4f7a-b7c5-b0604764c73c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.885653047Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.896409632Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ccf922d8-039a-4f7a-b7c5-b0604764c73c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.902206605Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.902981344Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.903084637Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.903135068Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.903172159Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.911653538Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:40.918133728Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.224110108Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions","method":"POST","req_id":"c2fa2fbd-ee30-4c23-9c3a-e0b34c954798","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":49,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.775597075Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c","method":"GET","req_id":"726eb7ea-a7d7-424f-96f2-67086b9ff7c9","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"path is \"/data/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.778420545Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"dataset is oxp_f50d55fe-477d-42a7-afdb-d417b796d9e5/crucible/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.789384415Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"region ccf922d8-039a-4f7a-b7c5-b0604764c73c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.799382572Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.799602887Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.799637968Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c","method":"DELETE","req_id":"1b3639f2-15fd-4702-b00d-0be79bb8ab87","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21267,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-ccf922d8-039a-4f7a-b7c5-b0604764c73c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.800759336Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.802573121Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c","method":"GET","req_id":"c8c602bd-3e9a-4782-984c-c5aa9f31e7f6","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.812490525Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f50d55fe-477d-42a7-afdb-d417b796d9e5/crucible/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.823303492Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"region":"ccf922d8-039a-4f7a-b7c5-b0604764c73c","component":"worker"}
{"msg":"region ccf922d8-039a-4f7a-b7c5-b0604764c73c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.908657398Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:41.917324752Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions/ccf922d8-039a-4f7a-b7c5-b0604764c73c","method":"GET","req_id":"a6df572d-41a5-42d9-9d95-7e870a05c01d","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"region b3bfc546-8331-4c8b-8641-152bf6da55d1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:43.975346712Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:43.975514996Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions","method":"POST","req_id":"f17fad05-187a-42e7-8c31-bb24be0ee053","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":235,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:43.975570898Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_f50d55fe-477d-42a7-afdb-d417b796d9e5/crucible/regions/b3bfc546-8331-4c8b-8641-152bf6da55d1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:43.984441677Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f50d55fe-477d-42a7-afdb-d417b796d9e5/crucible/regions/b3bfc546-8331-4c8b-8641-152bf6da55d1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:43.984472917Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b3bfc546-8331-4c8b-8641-152bf6da55d1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b3bfc546-8331-4c8b-8641-152bf6da55d1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.059413276Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"region":"b3bfc546-8331-4c8b-8641-152bf6da55d1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b3bfc546-8331-4c8b-8641-152bf6da55d1\" \"--data\" \"/data/regions/b3bfc546-8331-4c8b-8641-152bf6da55d1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.061618061Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"region":"b3bfc546-8331-4c8b-8641-152bf6da55d1","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.145160522Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"region":"b3bfc546-8331-4c8b-8641-152bf6da55d1","component":"worker"}
{"msg":"region b3bfc546-8331-4c8b-8641-152bf6da55d1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.145245814Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.145414978Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b3bfc546-8331-4c8b-8641-152bf6da55d1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.146790932Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b3bfc546-8331-4c8b-8641-152bf6da55d1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.150845723Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.158696036Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b3bfc546-8331-4c8b-8641-152bf6da55d1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.163659219Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b3bfc546-8331-4c8b-8641-152bf6da55d1","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.164010017Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.164071569Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.16411321Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.164159321Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.169842931Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.175749807Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-08T00:40:44.223132146Z","hostname":"oxz_crucible_8300c829-49f9-4839-892a-078272d50f4c","pid":24043,"uri":"/crucible/0/regions","method":"POST","req_id":"7fed223c-b00e-42f9-914f-cccf9d9f4b80","remote_addr":"[fd00:1122:3344:101::c]:64830","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":57,"response_code":"200"}