[ Dec 23 01:37:00 Enabled. ] [ Dec 23 01:37:00 Rereading configuration. ] [ Dec 23 01:37:02 Rereading configuration. ] [ Dec 23 01:37:04 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_b72b3fd7-3269-434b-88fe-93809c0271cf/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 7103cd3a3d7b0112d2949dd135db06fef0c156bb\nCommit timestamp: 2025-11-10T22:24:53.000000000Z branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:37:04.890729608Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996} {"msg":"dataset: \"oxp_b72b3fd7-3269-434b-88fe-93809c0271cf/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:37:04.89218559Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996} {"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:37:04.89219819Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:37:04.89220619Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:37:04.910179741Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:37:04.998226611Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.320941081Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:56201"} {"msg":"region 243754c3-ee96-4e2c-9027-99f85fbdae8d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.347744189Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.35060787Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.350654851Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions","method":"POST","req_id":"be294a83-10c9-4d4b-a6e9-a395d5c31691","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":14275,"response_code":200} {"msg":"zfs set reservation of 1342177280 for oxp_b72b3fd7-3269-434b-88fe-93809c0271cf/crucible/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.360886158Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_b72b3fd7-3269-434b-88fe-93809c0271cf/crucible/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.360906529Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"243754c3-ee96-4e2c-9027-99f85fbdae8d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.421029185Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"region":"243754c3-ee96-4e2c-9027-99f85fbdae8d","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"243754c3-ee96-4e2c-9027-99f85fbdae8d\" \"--data\" \"/data/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.423605539Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"region":"243754c3-ee96-4e2c-9027-99f85fbdae8d","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.609935853Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions","method":"POST","req_id":"8f17fcad-bdd0-48de-8165-10d8d553ae9d","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":70,"response_code":200} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.770815777Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"region":"243754c3-ee96-4e2c-9027-99f85fbdae8d","component":"worker"} {"msg":"region 243754c3-ee96-4e2c-9027-99f85fbdae8d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.770911649Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.771092803Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-243754c3-ee96-4e2c-9027-99f85fbdae8d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.77141742Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-243754c3-ee96-4e2c-9027-99f85fbdae8d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.776226132Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.79356783Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-243754c3-ee96-4e2c-9027-99f85fbdae8d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.797091635Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.79780563Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.797874731Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.797919272Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.797951853Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.802416978Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:01.806635907Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.328928566Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions","method":"POST","req_id":"f0591665-8b58-47b3-b35b-c6b3ad635a71","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":78,"response_code":200} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.748366497Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d","method":"GET","req_id":"633f6e3a-c8a2-4832-b1df-4fb6ccec9b2a","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":77,"response_code":200} {"msg":"path is \"/data/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.752208735Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"dataset is oxp_b72b3fd7-3269-434b-88fe-93809c0271cf/crucible/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.760534784Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"region 243754c3-ee96-4e2c-9027-99f85fbdae8d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.768467819Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.768635954Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.768674036Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d","method":"DELETE","req_id":"0a9afdb7-8ce6-4529-b48a-c93bd9129bc7","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":16454,"response_code":204} {"msg":"disabling downstairs instance: downstairs-243754c3-ee96-4e2c-9027-99f85fbdae8d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.7696637Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.773084343Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d","method":"GET","req_id":"b51af6e3-c4f1-44c7-a1b6-b332e9692e8f","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":27,"response_code":200} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.774383409Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_b72b3fd7-3269-434b-88fe-93809c0271cf/crucible/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.782454515Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"region":"243754c3-ee96-4e2c-9027-99f85fbdae8d","component":"worker"} {"msg":"region 243754c3-ee96-4e2c-9027-99f85fbdae8d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.851569344Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:02.893139501Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions/243754c3-ee96-4e2c-9027-99f85fbdae8d","method":"GET","req_id":"bd27edc7-e2ea-4fbe-982a-2099d0322f38","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":39,"response_code":200} {"msg":"region f2128925-c697-48f3-8b0e-3e5b9a784aa1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.383016463Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.383191142Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.38323543Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions","method":"POST","req_id":"4750cd09-5493-4d67-be77-b564cf2018db","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":249,"response_code":200} {"msg":"zfs set reservation of 2684354560 for oxp_b72b3fd7-3269-434b-88fe-93809c0271cf/crucible/regions/f2128925-c697-48f3-8b0e-3e5b9a784aa1","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.389024558Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_b72b3fd7-3269-434b-88fe-93809c0271cf/crucible/regions/f2128925-c697-48f3-8b0e-3e5b9a784aa1","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.389045672Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f2128925-c697-48f3-8b0e-3e5b9a784aa1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f2128925-c697-48f3-8b0e-3e5b9a784aa1\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.45282705Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"region":"f2128925-c697-48f3-8b0e-3e5b9a784aa1","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f2128925-c697-48f3-8b0e-3e5b9a784aa1\" \"--data\" \"/data/regions/f2128925-c697-48f3-8b0e-3e5b9a784aa1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.452886526Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"region":"f2128925-c697-48f3-8b0e-3e5b9a784aa1","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.526076659Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"region":"f2128925-c697-48f3-8b0e-3e5b9a784aa1","component":"worker"} {"msg":"region f2128925-c697-48f3-8b0e-3e5b9a784aa1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.526104803Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.526311086Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f2128925-c697-48f3-8b0e-3e5b9a784aa1","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.527774883Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f2128925-c697-48f3-8b0e-3e5b9a784aa1","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.530712076Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.539044225Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f2128925-c697-48f3-8b0e-3e5b9a784aa1","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.542258171Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f2128925-c697-48f3-8b0e-3e5b9a784aa1","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.542451254Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.542488988Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.542530726Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.542562771Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.546539754Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.551545658Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-12-23T01:38:05.707475081Z","hostname":"oxz_crucible_767f8173-ec60-4314-8bae-a61b50effa31","pid":21996,"uri":"/crucible/0/regions","method":"POST","req_id":"abe68419-f805-4320-a212-81ae6b49a74c","remote_addr":"[fd00:1122:3344:101::b]:56201","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":64,"response_code":200}