[ Mar 3 05:23:59 Enabled. ] [ Mar 3 05:23:59 Rereading configuration. ] [ Mar 3 05:24:01 Rereading configuration. ] [ Mar 3 05:24:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e6057776-3cf6-4214-a3ee-0a9da2e8db53/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_e6057776-3cf6-4214-a3ee-0a9da2e8db53/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:03.977938638Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012} {"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:03.981121952Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:03.981135302Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:04.001794356Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:04.10919791Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.386900789Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:54416"} {"msg":"region f37f13d1-975f-4f06-af3d-acfe5dc044c2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.431516184Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.434546395Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions","method":"POST","req_id":"13f91ff3-1499-45b5-bfc2-67dcc5147269","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":19809,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.434563885Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_e6057776-3cf6-4214-a3ee-0a9da2e8db53/crucible/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.443843721Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_e6057776-3cf6-4214-a3ee-0a9da2e8db53/crucible/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.443869112Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f37f13d1-975f-4f06-af3d-acfe5dc044c2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.516901227Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"region":"f37f13d1-975f-4f06-af3d-acfe5dc044c2","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f37f13d1-975f-4f06-af3d-acfe5dc044c2\" \"--data\" \"/data/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.519517559Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"region":"f37f13d1-975f-4f06-af3d-acfe5dc044c2","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.63029005Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"region":"f37f13d1-975f-4f06-af3d-acfe5dc044c2","component":"worker"} {"msg":"region f37f13d1-975f-4f06-af3d-acfe5dc044c2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.630335871Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.630428493Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f37f13d1-975f-4f06-af3d-acfe5dc044c2","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.630835121Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f37f13d1-975f-4f06-af3d-acfe5dc044c2","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.636176708Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.646290391Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f37f13d1-975f-4f06-af3d-acfe5dc044c2","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.649994385Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.650545077Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.650602788Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.650642129Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.650678679Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.655861883Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.66866408Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:48.747070112Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions","method":"POST","req_id":"9a48c938-a6b7-453a-9236-838a29fbb0c2","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.351288749Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","method":"GET","req_id":"b9f2db1d-3032-4f62-804f-920f85247c11","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":2252,"response_code":"200"} {"msg":"path is \"/data/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.353078305Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"dataset is oxp_e6057776-3cf6-4214-a3ee-0a9da2e8db53/crucible/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.364569025Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"region f37f13d1-975f-4f06-af3d-acfe5dc044c2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.374867492Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.374994284Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.375044915Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","method":"DELETE","req_id":"3d561a2d-b8c2-4f73-bf09-d902189b77ff","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":21982,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-f37f13d1-975f-4f06-af3d-acfe5dc044c2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.376043585Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.377802041Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","method":"GET","req_id":"520c6bd2-88fb-476b-8346-9d581738b284","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":24,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.380411293Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_e6057776-3cf6-4214-a3ee-0a9da2e8db53/crucible/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.390787231Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"region":"f37f13d1-975f-4f06-af3d-acfe5dc044c2","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.447325885Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","method":"GET","req_id":"3765e75f-37ee-40fe-9be1-fdaf28c3dc5e","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":78,"response_code":"200"} {"msg":"region f37f13d1-975f-4f06-af3d-acfe5dc044c2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.454974628Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:49.605198501Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions/f37f13d1-975f-4f06-af3d-acfe5dc044c2","method":"GET","req_id":"54af05ba-df69-435e-877e-92c3389c68bf","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":31,"response_code":"200"} {"msg":"region 8838cef8-54aa-452b-a7ee-0dc1979caed5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.736064464Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.736217737Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.736235247Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions","method":"POST","req_id":"0bda2012-5d7e-4d96-9ef7-c0595db52d80","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":214,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_e6057776-3cf6-4214-a3ee-0a9da2e8db53/crucible/regions/8838cef8-54aa-452b-a7ee-0dc1979caed5","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.744642216Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_e6057776-3cf6-4214-a3ee-0a9da2e8db53/crucible/regions/8838cef8-54aa-452b-a7ee-0dc1979caed5","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.744666236Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"8838cef8-54aa-452b-a7ee-0dc1979caed5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8838cef8-54aa-452b-a7ee-0dc1979caed5\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.817193401Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"region":"8838cef8-54aa-452b-a7ee-0dc1979caed5","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8838cef8-54aa-452b-a7ee-0dc1979caed5\" \"--data\" \"/data/regions/8838cef8-54aa-452b-a7ee-0dc1979caed5\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.819053208Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"region":"8838cef8-54aa-452b-a7ee-0dc1979caed5","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.911740667Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"region":"8838cef8-54aa-452b-a7ee-0dc1979caed5","component":"worker"} {"msg":"region 8838cef8-54aa-452b-a7ee-0dc1979caed5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.911787198Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.911878999Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-8838cef8-54aa-452b-a7ee-0dc1979caed5","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.913238787Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8838cef8-54aa-452b-a7ee-0dc1979caed5","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.915987642Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.924524033Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8838cef8-54aa-452b-a7ee-0dc1979caed5","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.927526293Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8838cef8-54aa-452b-a7ee-0dc1979caed5","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.927827469Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.927892381Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.927942582Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.927984533Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.931508233Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:51.936531454Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-03T05:24:52.102638895Z","hostname":"oxz_crucible_d5d0561a-d47e-4164-a2d5-567749a81b50","pid":22012,"uri":"/crucible/0/regions","method":"POST","req_id":"ebdabef6-cebb-4819-b9dd-ae55ab985550","remote_addr":"[fd00:1122:3344:101::b]:54416","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":62,"response_code":"200"}