[ Jan 22 09:21:46 Enabled. ]
[ Jan 22 09:21:46 Rereading configuration. ]
[ Jan 22 09:21:49 Rereading configuration. ]
[ Jan 22 09:21:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_34031fe8-4691-4be6-a0d4-bb1591795113/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_34031fe8-4691-4be6-a0d4-bb1591795113/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.137757806Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.138670596Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.138679626Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.156986363Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:21:53.229243168Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:28.822217357Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:44001"}
{"msg":"region 286f143a-feca-4d53-9659-54ff4ab487fd state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:28.840636015Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:28.842871049Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"uri":"/crucible/0/regions","method":"POST","req_id":"db2cd91a-e51b-4b38-8175-fae897f0a1b5","remote_addr":"[fd00:1122:3344:101::c]:44001","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":8975,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:28.843456305Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_34031fe8-4691-4be6-a0d4-bb1591795113/crucible/regions/286f143a-feca-4d53-9659-54ff4ab487fd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:28.852693943Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_34031fe8-4691-4be6-a0d4-bb1591795113/crucible/regions/286f143a-feca-4d53-9659-54ff4ab487fd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:28.852730554Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"286f143a-feca-4d53-9659-54ff4ab487fd\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/286f143a-feca-4d53-9659-54ff4ab487fd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:28.911828508Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"region":"286f143a-feca-4d53-9659-54ff4ab487fd","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"286f143a-feca-4d53-9659-54ff4ab487fd\" \"--data\" \"/data/regions/286f143a-feca-4d53-9659-54ff4ab487fd\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:28.911890989Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"region":"286f143a-feca-4d53-9659-54ff4ab487fd","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.047374244Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"region":"286f143a-feca-4d53-9659-54ff4ab487fd","component":"worker"}
{"msg":"region 286f143a-feca-4d53-9659-54ff4ab487fd state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.047426384Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.047501155Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-286f143a-feca-4d53-9659-54ff4ab487fd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.047899666Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-286f143a-feca-4d53-9659-54ff4ab487fd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.052470798Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.071980748Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-286f143a-feca-4d53-9659-54ff4ab487fd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.075808169Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/286f143a-feca-4d53-9659-54ff4ab487fd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.089210793Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.091276405Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.091336676Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.091390366Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.097644313Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.108023975Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.155906739Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"uri":"/crucible/0/regions","method":"POST","req_id":"795bbd8a-283d-4905-9265-f988436abfa5","remote_addr":"[fd00:1122:3344:101::c]:44001","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":64,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.495406154Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"uri":"/crucible/0/regions/286f143a-feca-4d53-9659-54ff4ab487fd","method":"GET","req_id":"dff1e214-f170-4105-90c8-d931e8d6153b","remote_addr":"[fd00:1122:3344:101::c]:44001","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"path is \"/data/regions/286f143a-feca-4d53-9659-54ff4ab487fd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.500879302Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"dataset is oxp_34031fe8-4691-4be6-a0d4-bb1591795113/crucible/regions/286f143a-feca-4d53-9659-54ff4ab487fd","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.512638468Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"region 286f143a-feca-4d53-9659-54ff4ab487fd state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.524421075Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.524598737Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-286f143a-feca-4d53-9659-54ff4ab487fd (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.525695239Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.526592008Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"uri":"/crucible/0/regions/286f143a-feca-4d53-9659-54ff4ab487fd","method":"DELETE","req_id":"be8af477-afcc-4871-a41c-67e28074027a","remote_addr":"[fd00:1122:3344:101::c]:44001","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25747,"response_code":"204"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.530834634Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_34031fe8-4691-4be6-a0d4-bb1591795113/crucible/regions/286f143a-feca-4d53-9659-54ff4ab487fd\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.54260212Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"region":"286f143a-feca-4d53-9659-54ff4ab487fd","component":"worker"}
{"msg":"region 286f143a-feca-4d53-9659-54ff4ab487fd state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.611691922Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:29.691104135Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"uri":"/crucible/0/regions/286f143a-feca-4d53-9659-54ff4ab487fd","method":"GET","req_id":"ed0a5382-d35e-40e1-b92e-7f29cbb13358","remote_addr":"[fd00:1122:3344:101::c]:44001","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":110,"response_code":"200"}
{"msg":"region 349ff87e-f838-4796-86d3-4215e7b4f6a6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.350551813Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.350693785Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.350704005Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"uri":"/crucible/0/regions","method":"POST","req_id":"329bc9e4-f789-42af-83ef-65c1598e9e4b","remote_addr":"[fd00:1122:3344:101::c]:44001","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":215,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_34031fe8-4691-4be6-a0d4-bb1591795113/crucible/regions/349ff87e-f838-4796-86d3-4215e7b4f6a6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.359309157Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_34031fe8-4691-4be6-a0d4-bb1591795113/crucible/regions/349ff87e-f838-4796-86d3-4215e7b4f6a6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.359346467Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"349ff87e-f838-4796-86d3-4215e7b4f6a6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/349ff87e-f838-4796-86d3-4215e7b4f6a6\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.410051792Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"region":"349ff87e-f838-4796-86d3-4215e7b4f6a6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"349ff87e-f838-4796-86d3-4215e7b4f6a6\" \"--data\" \"/data/regions/349ff87e-f838-4796-86d3-4215e7b4f6a6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.411992193Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"region":"349ff87e-f838-4796-86d3-4215e7b4f6a6","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.483203767Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"region":"349ff87e-f838-4796-86d3-4215e7b4f6a6","component":"worker"}
{"msg":"region 349ff87e-f838-4796-86d3-4215e7b4f6a6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.483253148Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.483359029Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-349ff87e-f838-4796-86d3-4215e7b4f6a6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.484879645Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-349ff87e-f838-4796-86d3-4215e7b4f6a6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.488197771Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.496090315Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-349ff87e-f838-4796-86d3-4215e7b4f6a6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.499598833Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/349ff87e-f838-4796-86d3-4215e7b4f6a6","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.499817366Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.499868966Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.499909216Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.499947507Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.504131702Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.509011944Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-22T09:22:32.664255671Z","hostname":"oxz_crucible_760efec7-e305-4d29-8d53-203649569f4a","pid":22519,"uri":"/crucible/0/regions","method":"POST","req_id":"3da6ff59-2ed0-45fc-9d3f-eab5b5693185","remote_addr":"[fd00:1122:3344:101::c]:44001","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":75,"response_code":"200"}