[ Dec 19 20:29:03 Enabled. ]
[ Dec 19 20:29:03 Rereading configuration. ]
[ Dec 19 20:29:08 Rereading configuration. ]
[ Dec 19 20:29:15 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f4fd3a6f-2f27-43a0-a52d-9a06f61d3fac/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f4fd3a6f-2f27-43a0-a52d-9a06f61d3fac/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:15.689543335Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:15.692899453Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:15.692909683Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:15.70824659Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:15.889034804Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.566765569Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:47330"}
{"msg":"region 6ae621f9-2353-401d-94cb-be3b0875ae8a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.586961718Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.591965045Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.597006982Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"uri":"/crucible/0/regions","method":"POST","req_id":"23263f4e-2fa2-45f5-b603-2bd2f6fb3b08","remote_addr":"[fd00:1122:3344:101::b]:47330","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":12154,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_f4fd3a6f-2f27-43a0-a52d-9a06f61d3fac/crucible/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.603873522Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f4fd3a6f-2f27-43a0-a52d-9a06f61d3fac/crucible/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.603922363Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6ae621f9-2353-401d-94cb-be3b0875ae8a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.690532597Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"region":"6ae621f9-2353-401d-94cb-be3b0875ae8a","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6ae621f9-2353-401d-94cb-be3b0875ae8a\" \"--data\" \"/data/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.692515703Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"region":"6ae621f9-2353-401d-94cb-be3b0875ae8a","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.915282523Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"uri":"/crucible/0/regions","method":"POST","req_id":"1cee704b-7e3d-448c-9930-2066b3874834","remote_addr":"[fd00:1122:3344:101::b]:47330","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":80,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.940451368Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"region":"6ae621f9-2353-401d-94cb-be3b0875ae8a","component":"worker"}
{"msg":"region 6ae621f9-2353-401d-94cb-be3b0875ae8a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.940508099Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.940733295Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6ae621f9-2353-401d-94cb-be3b0875ae8a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.941140894Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6ae621f9-2353-401d-94cb-be3b0875ae8a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.953514822Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.969397781Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6ae621f9-2353-401d-94cb-be3b0875ae8a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.97622092Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.976930426Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.976999768Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.977041919Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.97708725Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.983270914Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:36.994571567Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.483074386Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"uri":"/crucible/0/regions","method":"POST","req_id":"521572a3-fd43-4e5b-91f9-cdfab949be9b","remote_addr":"[fd00:1122:3344:101::b]:47330","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.784942615Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"uri":"/crucible/0/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a","method":"GET","req_id":"9ad02b44-9ab4-4bd0-9479-76c94ba1aad8","remote_addr":"[fd00:1122:3344:101::b]:47330","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":102,"response_code":"200"}
{"msg":"path is \"/data/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.78774631Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"datafile"}
{"msg":"dataset is oxp_f4fd3a6f-2f27-43a0-a52d-9a06f61d3fac/crucible/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.798222494Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"datafile"}
{"msg":"region 6ae621f9-2353-401d-94cb-be3b0875ae8a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.808939843Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.809088436Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.809173249Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"uri":"/crucible/0/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a","method":"DELETE","req_id":"efdeaa00-c98c-487a-8889-a073283803a7","remote_addr":"[fd00:1122:3344:101::b]:47330","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":21469,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-6ae621f9-2353-401d-94cb-be3b0875ae8a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.810170492Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.812144238Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"uri":"/crucible/0/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a","method":"GET","req_id":"60dbd38d-f301-4b81-b453-5a28cdfe8d59","remote_addr":"[fd00:1122:3344:101::b]:47330","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":37,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.815175718Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f4fd3a6f-2f27-43a0-a52d-9a06f61d3fac/crucible/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.826664065Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"region":"6ae621f9-2353-401d-94cb-be3b0875ae8a","component":"worker"}
{"msg":"region 6ae621f9-2353-401d-94cb-be3b0875ae8a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.924401158Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-19T20:29:37.947472305Z","hostname":"oxz_crucible_d9e7344e-5ca2-4201-a14a-40cbc674570c","pid":23617,"uri":"/crucible/0/regions/6ae621f9-2353-401d-94cb-be3b0875ae8a","method":"GET","req_id":"2415d9cf-5e4e-4729-9ad9-210b4ec3688e","remote_addr":"[fd00:1122:3344:101::b]:47330","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":82,"response_code":"200"}