[ Mar  4 03:17:41 Enabled. ]
[ Mar  4 03:17:41 Rereading configuration. ]
[ Mar  4 03:17:43 Rereading configuration. ]
[ Mar  4 03:17:56 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:17:56.35331983Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:17:56.356462648Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:17:56.356471928Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:17:56.37296226Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:17:56.485656271Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.213634336Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:51011"}
{"msg":"region 8522f663-bfd2-44db-b998-3fd36bc04757 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.258336474Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.261058457Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions","method":"POST","req_id":"eecdf989-af52-46f6-850f-1e50baad7078","remote_addr":"[fd00:1122:3344:101::b]:51011","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20074,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.261082518Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/8522f663-bfd2-44db-b998-3fd36bc04757","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.271646637Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/8522f663-bfd2-44db-b998-3fd36bc04757","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.271678327Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8522f663-bfd2-44db-b998-3fd36bc04757\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8522f663-bfd2-44db-b998-3fd36bc04757\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.338143382Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"8522f663-bfd2-44db-b998-3fd36bc04757","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8522f663-bfd2-44db-b998-3fd36bc04757\" \"--data\" \"/data/regions/8522f663-bfd2-44db-b998-3fd36bc04757\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.338212163Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"8522f663-bfd2-44db-b998-3fd36bc04757","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.575595371Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions","method":"POST","req_id":"2ed79d48-20ed-4d86-80b3-63f9abc4fcf4","remote_addr":"[fd00:1122:3344:101::b]:51011","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.633277728Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"8522f663-bfd2-44db-b998-3fd36bc04757","component":"worker"}
{"msg":"region 8522f663-bfd2-44db-b998-3fd36bc04757 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.633320809Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.63341836Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8522f663-bfd2-44db-b998-3fd36bc04757","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.633847265Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8522f663-bfd2-44db-b998-3fd36bc04757","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.639765228Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.656404202Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8522f663-bfd2-44db-b998-3fd36bc04757","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.659768313Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8522f663-bfd2-44db-b998-3fd36bc04757","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.660753305Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.660848916Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.660905737Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.660954987Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.665508813Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.680440026Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:43.886534512Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions","method":"POST","req_id":"1a2cbc1c-9ce6-423d-8edc-95b0b7224006","remote_addr":"[fd00:1122:3344:101::b]:51011","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":52,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.634505499Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/8522f663-bfd2-44db-b998-3fd36bc04757","method":"GET","req_id":"d1ce4691-affd-41c1-b4c5-41a9d3a49956","remote_addr":"[fd00:1122:3344:101::b]:51011","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":109,"response_code":"200"}
{"msg":"path is \"/data/regions/8522f663-bfd2-44db-b998-3fd36bc04757\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.637446375Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"dataset is oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/8522f663-bfd2-44db-b998-3fd36bc04757","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.648341419Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"region 8522f663-bfd2-44db-b998-3fd36bc04757 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.658709176Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.658868388Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.658910058Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/8522f663-bfd2-44db-b998-3fd36bc04757","method":"DELETE","req_id":"1ac0c7d3-4ae4-47ec-a22f-9e2b0dd47428","remote_addr":"[fd00:1122:3344:101::b]:51011","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21511,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-8522f663-bfd2-44db-b998-3fd36bc04757 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.660032042Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.661419369Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/8522f663-bfd2-44db-b998-3fd36bc04757","method":"GET","req_id":"1829b7e1-9055-4de4-ab8f-3499ba3400a4","remote_addr":"[fd00:1122:3344:101::b]:51011","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":30,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.670640392Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/8522f663-bfd2-44db-b998-3fd36bc04757\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.682391446Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"8522f663-bfd2-44db-b998-3fd36bc04757","component":"worker"}
{"msg":"region 8522f663-bfd2-44db-b998-3fd36bc04757 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.749026173Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:18:44.803013444Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/8522f663-bfd2-44db-b998-3fd36bc04757","method":"GET","req_id":"a86c964d-0eaa-4e4c-b4f9-0f11cea73650","remote_addr":"[fd00:1122:3344:101::b]:51011","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.638177062Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:40323"}
{"msg":"region 0a9fc6e0-c356-4a38-ab20-966e3292009a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.638288673Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.638921451Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.638934791Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions","method":"POST","req_id":"d07b5305-da79-4b47-be36-32bce5f8c534","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":673,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/0a9fc6e0-c356-4a38-ab20-966e3292009a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.648389997Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/0a9fc6e0-c356-4a38-ab20-966e3292009a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.648425698Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0a9fc6e0-c356-4a38-ab20-966e3292009a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0a9fc6e0-c356-4a38-ab20-966e3292009a\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.703720005Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"0a9fc6e0-c356-4a38-ab20-966e3292009a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0a9fc6e0-c356-4a38-ab20-966e3292009a\" \"--data\" \"/data/regions/0a9fc6e0-c356-4a38-ab20-966e3292009a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.70572285Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"0a9fc6e0-c356-4a38-ab20-966e3292009a","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.796581303Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"0a9fc6e0-c356-4a38-ab20-966e3292009a","component":"worker"}
{"msg":"region 0a9fc6e0-c356-4a38-ab20-966e3292009a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.796630534Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.796757386Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0a9fc6e0-c356-4a38-ab20-966e3292009a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.798557838Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0a9fc6e0-c356-4a38-ab20-966e3292009a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.805443812Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.821212185Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0a9fc6e0-c356-4a38-ab20-966e3292009a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.824541836Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0a9fc6e0-c356-4a38-ab20-966e3292009a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.824827439Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.82489758Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.824958781Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.825020392Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.828891499Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.834828132Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:02.974018318Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions","method":"POST","req_id":"99e70387-39c0-44dc-8812-1ae4b885602f","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region 1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.611731522Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.611999536Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.612059637Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions","method":"POST","req_id":"ee778235-9853-4882-9a53-c123da0a420d","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":307,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.622730827Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.622765858Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.679041267Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad\" \"--data\" \"/data/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.679103598Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.754546323Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions","method":"POST","req_id":"a48f126d-a20e-40ca-9e2b-d3b411f8651d","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.768924359Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","component":"worker"}
{"msg":"region 1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.76896594Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.769148472Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.77148694Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.783451357Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.79345652Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.796536497Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.796848481Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.796900092Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.796944112Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.796987363Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.801278445Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:08.807892337Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:09.068529961Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:47981"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:09.069377951Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","method":"GET","req_id":"7f5ad412-f2fa-4578-b9dd-e1f25f7bd1ad","remote_addr":"[fd00:1122:3344:101::b]:47981","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":770,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:09.222866552Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions","method":"POST","req_id":"340e528d-fd19-40d9-99c0-d521d57df6c4","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.025058944Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","method":"GET","req_id":"c5a3d68f-894a-4a0b-a81a-d370bd6acf9d","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":41,"response_code":"200"}
{"msg":"path is \"/data/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.028423115Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"dataset is oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.041717188Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"region 1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.051826662Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.052073555Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.052145616Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","method":"DELETE","req_id":"280d06bb-e676-4cb6-9ab8-ddde4274852a","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23751,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.053768246Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.055089742Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","method":"GET","req_id":"dea80bed-c861-49d5-a972-c1c6e447f7e8","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.059551527Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8bf53b00-dedd-4516-be68-25e52e3985f5/crucible/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.070974557Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"region":"1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.111603125Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","method":"GET","req_id":"9dcbb5e7-41ec-4c58-b0e5-752855bfe5a1","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":685,"response_code":"200"}
{"msg":"region 1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.129319832Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-04T03:29:35.209613796Z","hostname":"oxz_crucible_382dd280-976a-45c5-b45f-5de9952f556b","pid":23567,"uri":"/crucible/0/regions/1af7cf48-ae5e-4bc9-84fd-edd57f0f89ad","method":"GET","req_id":"6951f3bc-2759-49d0-b8f4-47b052910191","remote_addr":"[fd00:1122:3344:101::b]:40323","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":27,"response_code":"200"}