[ Nov 7 02:43:12 Enabled. ] [ Nov 7 02:43:12 Rereading configuration. ] [ Nov 7 02:43:15 Rereading configuration. ] [ Nov 7 02:43:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_dddcfc0e-109d-4074-b924-e7a90aa78ab4/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_dddcfc0e-109d-4074-b924-e7a90aa78ab4/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:17.334447367Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928} {"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:17.337520378Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:17.337536649Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:17.357744305Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:17.496579038Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.032314782Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:56111"} {"msg":"region e72bb3d0-c81f-44b2-b3f8-a2b82ddef458 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.039204726Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.039542182Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions","method":"POST","req_id":"e3621651-facf-4103-a7f0-c4c36b198dc0","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2644,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.040857774Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_dddcfc0e-109d-4074-b924-e7a90aa78ab4/crucible/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.051061964Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_dddcfc0e-109d-4074-b924-e7a90aa78ab4/crucible/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.051145855Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"e72bb3d0-c81f-44b2-b3f8-a2b82ddef458\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.11865532Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"region":"e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e72bb3d0-c81f-44b2-b3f8-a2b82ddef458\" \"--data\" \"/data/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.120422979Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"region":"e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.241518816Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"region":"e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","component":"worker"} {"msg":"region e72bb3d0-c81f-44b2-b3f8-a2b82ddef458 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.241591568Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.24173823Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.242129307Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.245562804Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.253906843Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.257483122Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.258191064Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.258284226Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.258336417Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.258378677Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.26275588Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.267657332Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.406598706Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions","method":"POST","req_id":"4173d040-9348-453e-8075-b3d02a5aea82","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":62,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.643354221Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","method":"GET","req_id":"77ac312e-ddab-4e1e-a139-7eab0f06888f","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":87,"response_code":"200"} {"msg":"path is \"/data/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.64872047Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"dataset is oxp_dddcfc0e-109d-4074-b924-e7a90aa78ab4/crucible/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.659575321Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"region e72bb3d0-c81f-44b2-b3f8-a2b82ddef458 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.669510246Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.6697107Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.669775471Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","method":"DELETE","req_id":"06d41b35-9a2f-422d-80ae-b484ac080de9","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":21076,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-e72bb3d0-c81f-44b2-b3f8-a2b82ddef458 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.670781338Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.674992288Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","method":"GET","req_id":"f5d62950-01ca-4982-b092-3f697b1c3cc0","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":36,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.676180217Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_dddcfc0e-109d-4074-b924-e7a90aa78ab4/crucible/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.687674859Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"region":"e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","component":"worker"} {"msg":"region e72bb3d0-c81f-44b2-b3f8-a2b82ddef458 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.756504435Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:41.769728596Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions/e72bb3d0-c81f-44b2-b3f8-a2b82ddef458","method":"GET","req_id":"32383403-04a0-44e0-837e-85925805e2c4","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":66,"response_code":"200"} {"msg":"region f25cc0f1-38c5-4443-a1a3-9110f8a93f65 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.684320789Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.684517112Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.684534042Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions","method":"POST","req_id":"edd0ef65-1415-4071-a813-d9658d167712","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":271,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_dddcfc0e-109d-4074-b924-e7a90aa78ab4/crucible/regions/f25cc0f1-38c5-4443-a1a3-9110f8a93f65","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.693043104Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_dddcfc0e-109d-4074-b924-e7a90aa78ab4/crucible/regions/f25cc0f1-38c5-4443-a1a3-9110f8a93f65","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.693111075Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f25cc0f1-38c5-4443-a1a3-9110f8a93f65\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f25cc0f1-38c5-4443-a1a3-9110f8a93f65\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.759504041Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"region":"f25cc0f1-38c5-4443-a1a3-9110f8a93f65","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f25cc0f1-38c5-4443-a1a3-9110f8a93f65\" \"--data\" \"/data/regions/f25cc0f1-38c5-4443-a1a3-9110f8a93f65\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.761554785Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"region":"f25cc0f1-38c5-4443-a1a3-9110f8a93f65","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.822789295Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions","method":"POST","req_id":"437692b3-95a6-4a7b-8a9a-8908b2ebc20b","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":65,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.823996826Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"region":"f25cc0f1-38c5-4443-a1a3-9110f8a93f65","component":"worker"} {"msg":"region f25cc0f1-38c5-4443-a1a3-9110f8a93f65 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.824038626Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.82426863Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f25cc0f1-38c5-4443-a1a3-9110f8a93f65","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.825683903Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f25cc0f1-38c5-4443-a1a3-9110f8a93f65","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.828702974Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.835737711Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f25cc0f1-38c5-4443-a1a3-9110f8a93f65","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.839126367Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f25cc0f1-38c5-4443-a1a3-9110f8a93f65","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.839365022Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.839416822Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.839456753Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.839500034Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.843203586Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:43.848180498Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-07T02:43:44.406626961Z","hostname":"oxz_crucible_c66c4807-87bf-49f2-97cb-31d8d688e481","pid":22928,"uri":"/crucible/0/regions","method":"POST","req_id":"77047c4c-3da8-405e-8bb5-52323e10e1e6","remote_addr":"[fd00:1122:3344:101::a]:56111","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":52,"response_code":"200"}