[ Nov  8 00:00:03 Enabled. ]
[ Nov  8 00:00:03 Rereading configuration. ]
[ Nov  8 00:00:06 Rereading configuration. ]
[ Nov  8 00:00:11 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fbe96c0b-3c50-49b6-828c-12beaacd3aba/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 75ea3408643664877aa603ffc65996ed6dc34c0a\nCommit timestamp: 2025-10-24T15:21:57.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:00:11.318723792Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820}
{"msg":"dataset: \"oxp_fbe96c0b-3c50-49b6-828c-12beaacd3aba/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:00:11.319597386Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:00:11.319625647Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:00:11.319634427Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:00:11.337671213Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:00:11.466357709Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.576068642Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:50133"}
{"msg":"region d78c3bd3-52f6-45a9-b7b2-014103cedd6f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.626753935Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.634080186Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.634100307Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"uri":"/crucible/0/regions","method":"POST","req_id":"fd9ec1fb-59fa-421e-ac0c-05d04f785b26","remote_addr":"[fd00:1122:3344:101::c]:50133","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27459,"response_code":200}
{"msg":"zfs set reservation of 2684354560 for oxp_fbe96c0b-3c50-49b6-828c-12beaacd3aba/crucible/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.645740777Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_fbe96c0b-3c50-49b6-828c-12beaacd3aba/crucible/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.645758407Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d78c3bd3-52f6-45a9-b7b2-014103cedd6f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.697747506Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"region":"d78c3bd3-52f6-45a9-b7b2-014103cedd6f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d78c3bd3-52f6-45a9-b7b2-014103cedd6f\" \"--data\" \"/data/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.700247404Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"region":"d78c3bd3-52f6-45a9-b7b2-014103cedd6f","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:28.947236941Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"uri":"/crucible/0/regions","method":"POST","req_id":"4a08df18-8622-4512-87e7-5951c97a5ede","remote_addr":"[fd00:1122:3344:101::c]:50133","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":62,"response_code":200}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.030405617Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"region":"d78c3bd3-52f6-45a9-b7b2-014103cedd6f","component":"worker"}
{"msg":"region d78c3bd3-52f6-45a9-b7b2-014103cedd6f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.030453218Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.03053168Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d78c3bd3-52f6-45a9-b7b2-014103cedd6f","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.031007783Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d78c3bd3-52f6-45a9-b7b2-014103cedd6f","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.046240481Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.07021039Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d78c3bd3-52f6-45a9-b7b2-014103cedd6f","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.07458471Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.077551632Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.077620824Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.077668635Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.077714006Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.081753157Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.096826811Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:29.675454931Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"uri":"/crucible/0/regions","method":"POST","req_id":"9e351ce6-b586-4902-9702-0a79f85b32bd","remote_addr":"[fd00:1122:3344:101::c]:50133","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":61,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.504507208Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"uri":"/crucible/0/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f","method":"GET","req_id":"2914c65e-d817-4b29-ab10-8b27e4d87727","remote_addr":"[fd00:1122:3344:101::c]:50133","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":146,"response_code":200}
{"msg":"path is \"/data/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.507520161Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"datafile"}
{"msg":"dataset is oxp_fbe96c0b-3c50-49b6-828c-12beaacd3aba/crucible/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.515350646Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"datafile"}
{"msg":"region d78c3bd3-52f6-45a9-b7b2-014103cedd6f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.522822771Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.522967435Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"uri":"/crucible/0/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f","method":"DELETE","req_id":"93e4ff50-f4e7-426a-b003-47036d460491","remote_addr":"[fd00:1122:3344:101::c]:50133","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":15469,"response_code":204}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.522993916Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-d78c3bd3-52f6-45a9-b7b2-014103cedd6f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.524152998Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.527748676Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"uri":"/crucible/0/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f","method":"GET","req_id":"81f57870-1e88-4ae5-bd5b-b02923dc22ad","remote_addr":"[fd00:1122:3344:101::c]:50133","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":31,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.533481004Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fbe96c0b-3c50-49b6-828c-12beaacd3aba/crucible/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f\"","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.540520238Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"region":"d78c3bd3-52f6-45a9-b7b2-014103cedd6f","component":"worker"}
{"msg":"region d78c3bd3-52f6-45a9-b7b2-014103cedd6f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.59121938Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-11-08T00:11:54.601865113Z","hostname":"oxz_crucible_333b4388-4f58-4b47-8169-80b5d004daf8","pid":27820,"uri":"/crucible/0/regions/d78c3bd3-52f6-45a9-b7b2-014103cedd6f","method":"GET","req_id":"4e1fdcc1-0f60-47aa-93c7-eadefdc23071","remote_addr":"[fd00:1122:3344:101::c]:50133","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":34,"response_code":200}