[ Jun 3 22:45:04 Enabled. ] [ Jun 3 22:45:04 Rereading configuration. ] [ Jun 3 22:45:05 Rereading configuration. ] [ Jun 3 22:45:08 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_5abdebae-627b-424e-a85e-9f02a3167941/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: bd9a0e2abe6b6b89aec8c85f4ee57474144ed150\nCommit timestamp: 2026-05-14T15:48:00.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:45:08.889118847Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249} {"msg":"dataset: \"oxp_5abdebae-627b-424e-a85e-9f02a3167941/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:45:08.889315697Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249} {"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:45:08.889327641Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:45:08.889334504Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:45:08.910110399Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:45:08.96941688Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.670183451Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64895"} {"msg":"region 5666e36d-557e-453a-a726-d88f59ead101 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.704060018Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.708626752Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.708646603Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"uri":"/crucible/0/regions","method":"POST","req_id":"46400043-da99-4bd4-af0f-2a653a238972","remote_addr":"[fd00:1122:3344:101::b]:64895","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":13292,"response_code":200} {"msg":"zfs set reservation of 1342177280 for oxp_5abdebae-627b-424e-a85e-9f02a3167941/crucible/regions/5666e36d-557e-453a-a726-d88f59ead101","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.716541605Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_5abdebae-627b-424e-a85e-9f02a3167941/crucible/regions/5666e36d-557e-453a-a726-d88f59ead101","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.716563466Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"5666e36d-557e-453a-a726-d88f59ead101\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5666e36d-557e-453a-a726-d88f59ead101\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.767342501Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"region":"5666e36d-557e-453a-a726-d88f59ead101","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5666e36d-557e-453a-a726-d88f59ead101\" \"--data\" \"/data/regions/5666e36d-557e-453a-a726-d88f59ead101\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.769403043Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"region":"5666e36d-557e-453a-a726-d88f59ead101","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.889407294Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"region":"5666e36d-557e-453a-a726-d88f59ead101","component":"worker"} {"msg":"region 5666e36d-557e-453a-a726-d88f59ead101 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.889461715Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.889557687Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-5666e36d-557e-453a-a726-d88f59ead101","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.891831764Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5666e36d-557e-453a-a726-d88f59ead101","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.89601085Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.906073907Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5666e36d-557e-453a-a726-d88f59ead101","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.907605679Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5666e36d-557e-453a-a726-d88f59ead101","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.908677821Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.908702271Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.908719832Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.908726502Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.921382422Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:30.924245561Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.009983467Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"uri":"/crucible/0/regions","method":"POST","req_id":"078961e5-6308-4f66-8bea-54480c003f1f","remote_addr":"[fd00:1122:3344:101::b]:64895","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":165,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.262079417Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"uri":"/crucible/0/regions/5666e36d-557e-453a-a726-d88f59ead101","method":"GET","req_id":"368cffff-d748-45f4-89a2-ee0fcec5dd6f","remote_addr":"[fd00:1122:3344:101::b]:64895","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":77,"response_code":200} {"msg":"path is \"/data/regions/5666e36d-557e-453a-a726-d88f59ead101\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.265192601Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"dataset is oxp_5abdebae-627b-424e-a85e-9f02a3167941/crucible/regions/5666e36d-557e-453a-a726-d88f59ead101","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.272748616Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"region 5666e36d-557e-453a-a726-d88f59ead101 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.280053907Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.280189559Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"uri":"/crucible/0/regions/5666e36d-557e-453a-a726-d88f59ead101","method":"DELETE","req_id":"3f9eecc1-0fe8-446f-a047-2237cede632e","remote_addr":"[fd00:1122:3344:101::b]:64895","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":15012,"response_code":204} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.28022532Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-5666e36d-557e-453a-a726-d88f59ead101 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.281137849Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.28264742Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"uri":"/crucible/0/regions/5666e36d-557e-453a-a726-d88f59ead101","method":"GET","req_id":"f10ea568-e18c-43f2-81f8-4432bfbb2cad","remote_addr":"[fd00:1122:3344:101::b]:64895","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":63,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.283450777Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_5abdebae-627b-424e-a85e-9f02a3167941/crucible/regions/5666e36d-557e-453a-a726-d88f59ead101\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.291975102Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"region":"5666e36d-557e-453a-a726-d88f59ead101","component":"worker"} {"msg":"region 5666e36d-557e-453a-a726-d88f59ead101 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.349917725Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:46:31.434602218Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"uri":"/crucible/0/regions/5666e36d-557e-453a-a726-d88f59ead101","method":"GET","req_id":"9f473839-89c3-438c-8b5a-09343a97e60d","remote_addr":"[fd00:1122:3344:101::b]:64895","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":65,"response_code":200} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.754642155Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:32873"} {"msg":"region 745dc338-f6ed-454a-b10c-f4d6a9dbd1ff state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.754984222Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.759427483Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.759449154Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"uri":"/crucible/0/regions","method":"POST","req_id":"c3e5dd7f-f082-46ad-a5ae-c847e02b7731","remote_addr":"[fd00:1122:3344:101::b]:32873","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":4709,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_5abdebae-627b-424e-a85e-9f02a3167941/crucible/regions/745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.765253293Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_5abdebae-627b-424e-a85e-9f02a3167941/crucible/regions/745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.765282364Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"745dc338-f6ed-454a-b10c-f4d6a9dbd1ff\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/745dc338-f6ed-454a-b10c-f4d6a9dbd1ff\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.814088599Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"region":"745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"745dc338-f6ed-454a-b10c-f4d6a9dbd1ff\" \"--data\" \"/data/regions/745dc338-f6ed-454a-b10c-f4d6a9dbd1ff\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.81415088Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"region":"745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.867611701Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"region":"745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","component":"worker"} {"msg":"region 745dc338-f6ed-454a-b10c-f4d6a9dbd1ff state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.867648851Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.867753764Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.8685301Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.87824913Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.892034233Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.89330269Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/745dc338-f6ed-454a-b10c-f4d6a9dbd1ff","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.893504264Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.893543095Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.893575135Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.893617576Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.895298431Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:39.898283572Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-06-03T22:57:40.112134305Z","hostname":"oxz_crucible_dda1cac3-354b-46ae-b9d2-76beb162179f","pid":26249,"uri":"/crucible/0/regions","method":"POST","req_id":"2017e386-87c6-4a2b-9c9e-2638716056a9","remote_addr":"[fd00:1122:3344:101::b]:32873","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":67,"response_code":200}