[ Sep 15 21:05:50 Enabled. ]
[ Sep 15 21:05:50 Rereading configuration. ]
[ Sep 15 21:05:51 Rereading configuration. ]
[ Sep 15 21:05:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8c604df3-884d-443b-882a-a35215b4e1ad/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ad8a31742adc45e925e63443a5b43c8e30604022\nCommit timestamp: 2026-07-28T19:41:59.000000000Z  branch: main\nrustc: 1.90.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: false Opt level: 3","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:05:53.801031832Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669}
{"msg":"dataset: \"oxp_8c604df3-884d-443b-882a-a35215b4e1ad/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:05:53.801195946Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:05:53.801212426Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:05:53.801218236Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:05:53.820482232Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:05:53.903546014Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.06318574Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:37462"}
{"msg":"region 50ec95b9-783e-4320-90bc-4d0d1bb4a055 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.09537574Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.100654137Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.105107515Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"uri":"/crucible/0/regions","method":"POST","req_id":"56c46405-7be3-45be-983b-df42c29c56df","remote_addr":"[fd00:1122:3344:101::c]:37462","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":16595,"response_code":200}
{"msg":"zfs set reservation of 1342177280 for oxp_8c604df3-884d-443b-882a-a35215b4e1ad/crucible/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.110060924Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_8c604df3-884d-443b-882a-a35215b4e1ad/crucible/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.110084075Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"50ec95b9-783e-4320-90bc-4d0d1bb4a055\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.173074925Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"region":"50ec95b9-783e-4320-90bc-4d0d1bb4a055","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"50ec95b9-783e-4320-90bc-4d0d1bb4a055\" \"--data\" \"/data/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.175185891Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"region":"50ec95b9-783e-4320-90bc-4d0d1bb4a055","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.303631586Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"region":"50ec95b9-783e-4320-90bc-4d0d1bb4a055","component":"worker"}
{"msg":"region 50ec95b9-783e-4320-90bc-4d0d1bb4a055 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.307508212Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.30788442Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-50ec95b9-783e-4320-90bc-4d0d1bb4a055","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.310270473Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-50ec95b9-783e-4320-90bc-4d0d1bb4a055","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.316997221Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.329650271Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-50ec95b9-783e-4320-90bc-4d0d1bb4a055","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.331472501Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.332212057Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.332285199Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.33234123Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.332373471Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.335102131Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.338437225Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:51.392328854Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"uri":"/crucible/0/regions","method":"POST","req_id":"6bc00bc4-df44-4627-9710-0796045aa737","remote_addr":"[fd00:1122:3344:101::c]:37462","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":76,"response_code":200}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.175824995Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"uri":"/crucible/0/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055","method":"GET","req_id":"1bae9866-0853-4622-9d82-e201a9026331","remote_addr":"[fd00:1122:3344:101::c]:37462","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":79,"response_code":200}
{"msg":"path is \"/data/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.181127343Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"dataset is oxp_8c604df3-884d-443b-882a-a35215b4e1ad/crucible/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.188232899Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"region 50ec95b9-783e-4320-90bc-4d0d1bb4a055 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.195439828Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.195621682Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.195632252Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"uri":"/crucible/0/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055","method":"DELETE","req_id":"9146f896-d14f-4eac-a167-d302d33d6e67","remote_addr":"[fd00:1122:3344:101::c]:37462","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":14548,"response_code":204}
{"msg":"disabling downstairs instance: downstairs-50ec95b9-783e-4320-90bc-4d0d1bb4a055 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.196597064Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.19912487Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"uri":"/crucible/0/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055","method":"GET","req_id":"b3589d4d-f9b4-4374-90e0-ce618b45002a","remote_addr":"[fd00:1122:3344:101::c]:37462","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":66,"response_code":200}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.199518808Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8c604df3-884d-443b-882a-a35215b4e1ad/crucible/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.208117198Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"region":"50ec95b9-783e-4320-90bc-4d0d1bb4a055","component":"worker"}
{"msg":"region 50ec95b9-783e-4320-90bc-4d0d1bb4a055 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.273108272Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:52.306272434Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"uri":"/crucible/0/regions/50ec95b9-783e-4320-90bc-4d0d1bb4a055","method":"GET","req_id":"3a7ce1c9-97e4-41b8-9538-a9cc251503e3","remote_addr":"[fd00:1122:3344:101::c]:37462","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":49,"response_code":200}
{"msg":"region 9185abcf-9829-40ef-8c23-b99df80a0965 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.043085763Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.043312998Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"uri":"/crucible/0/regions","method":"POST","req_id":"b06dc24d-d5b7-4706-b5a7-00026beed578","remote_addr":"[fd00:1122:3344:101::c]:37462","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":276,"response_code":200}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.04341298Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_8c604df3-884d-443b-882a-a35215b4e1ad/crucible/regions/9185abcf-9829-40ef-8c23-b99df80a0965","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.050335243Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_8c604df3-884d-443b-882a-a35215b4e1ad/crucible/regions/9185abcf-9829-40ef-8c23-b99df80a0965","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.050360584Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9185abcf-9829-40ef-8c23-b99df80a0965\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9185abcf-9829-40ef-8c23-b99df80a0965\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.114970209Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"region":"9185abcf-9829-40ef-8c23-b99df80a0965","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9185abcf-9829-40ef-8c23-b99df80a0965\" \"--data\" \"/data/regions/9185abcf-9829-40ef-8c23-b99df80a0965\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.115051781Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"region":"9185abcf-9829-40ef-8c23-b99df80a0965","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.165090246Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"region":"9185abcf-9829-40ef-8c23-b99df80a0965","component":"worker"}
{"msg":"region 9185abcf-9829-40ef-8c23-b99df80a0965 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.165171597Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.165358212Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9185abcf-9829-40ef-8c23-b99df80a0965","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.166593219Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9185abcf-9829-40ef-8c23-b99df80a0965","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.16801783Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.172253274Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9185abcf-9829-40ef-8c23-b99df80a0965","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.173649164Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9185abcf-9829-40ef-8c23-b99df80a0965","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.173936611Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.173988732Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.174026133Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.174071544Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.176323094Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.179678528Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2026-09-15T21:06:54.410279357Z","hostname":"oxz_crucible_0ff86dc6-9995-4e21-b126-c97846bd57e7","pid":22669,"uri":"/crucible/0/regions","method":"POST","req_id":"52d83859-1db4-486b-bc6c-a501d0ed333a","remote_addr":"[fd00:1122:3344:101::c]:37462","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":58,"response_code":200}