[ Feb 21 14:32:18 Enabled. ]
[ Feb 21 14:32:18 Rereading configuration. ]
[ Feb 21 14:32:24 Rereading configuration. ]
[ Feb 21 14:32:30 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:32:30.837915284Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:32:30.841166061Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:32:30.841174521Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:32:30.860490569Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:32:30.931127213Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.319788088Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:33056"}
{"msg":"region 014fde38-0cd8-400c-b2a3-629d2289e4c6 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.348239329Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.351063724Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions","method":"POST","req_id":"cb407fdf-1f32-4e29-8df9-e9ad05047235","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":13842,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.351083664Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.361417433Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.361451254Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"014fde38-0cd8-400c-b2a3-629d2289e4c6\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.418630946Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"014fde38-0cd8-400c-b2a3-629d2289e4c6","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"014fde38-0cd8-400c-b2a3-629d2289e4c6\" \"--data\" \"/data/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.420500211Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"014fde38-0cd8-400c-b2a3-629d2289e4c6","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.566896903Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions","method":"POST","req_id":"8e278a38-ed2d-4ebf-a1e2-8a9886be5388","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.671201814Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"014fde38-0cd8-400c-b2a3-629d2289e4c6","component":"worker"}
{"msg":"region 014fde38-0cd8-400c-b2a3-629d2289e4c6 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.671249455Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.671360799Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-014fde38-0cd8-400c-b2a3-629d2289e4c6","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.671838473Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-014fde38-0cd8-400c-b2a3-629d2289e4c6","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.678251825Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.68944819Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-014fde38-0cd8-400c-b2a3-629d2289e4c6","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.69246307Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.693864182Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.696206993Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.696261144Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.696299745Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.707767098Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:22.712425408Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.297062354Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions","method":"POST","req_id":"ecb3fb55-1f9a-4857-b424-e841ab4d3dba","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":44,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.606675979Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6","method":"GET","req_id":"08864495-3680-4c23-8205-e6649ea44e65","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"path is \"/data/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.609353009Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"dataset is oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.62039388Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"region 014fde38-0cd8-400c-b2a3-629d2289e4c6 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.629995837Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.630140291Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-014fde38-0cd8-400c-b2a3-629d2289e4c6 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.631190803Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.6320759Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6","method":"DELETE","req_id":"f6de6453-6307-42fd-a1e0-bb98dac6dd69","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22773,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.634493762Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6","method":"GET","req_id":"f1a59fb8-007e-456f-b45b-da1c7be8af78","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.636201883Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.646645936Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"014fde38-0cd8-400c-b2a3-629d2289e4c6","component":"worker"}
{"msg":"region 014fde38-0cd8-400c-b2a3-629d2289e4c6 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.704137226Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:23.781989306Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions/014fde38-0cd8-400c-b2a3-629d2289e4c6","method":"GET","req_id":"dcb85fb2-e101-4422-b087-e44cdcb2e0b1","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region 02e8e590-f018-4ea9-900c-3f16aae4fb14 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.679260774Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.681504832Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.681527922Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions","method":"POST","req_id":"873a1b67-4558-4561-9c16-06df4b8e7d03","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":2304,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.691029497Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.691064148Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"02e8e590-f018-4ea9-900c-3f16aae4fb14\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.738716074Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"02e8e590-f018-4ea9-900c-3f16aae4fb14","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"02e8e590-f018-4ea9-900c-3f16aae4fb14\" \"--data\" \"/data/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.740666892Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"02e8e590-f018-4ea9-900c-3f16aae4fb14","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.818154251Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"02e8e590-f018-4ea9-900c-3f16aae4fb14","component":"worker"}
{"msg":"region 02e8e590-f018-4ea9-900c-3f16aae4fb14 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.818196063Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.818297946Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.819702757Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.82313416Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.83247915Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.835366806Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.835659055Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.835718927Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.835761978Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.835803909Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.839795819Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.845556681Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:33:25.930345529Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions","method":"POST","req_id":"996803d3-98c7-4eda-a737-991187bc59ae","remote_addr":"[fd00:1122:3344:101::a]:33056","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.087623619Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47087"}
{"msg":"region ff14cda9-79b4-4a96-8756-5ce365d2398d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.087779063Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.088356401Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.088371471Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions","method":"POST","req_id":"f9a9cfaa-e351-40fa-8909-b6d1461bfdc6","remote_addr":"[fd00:1122:3344:101::a]:47087","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":678,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/ff14cda9-79b4-4a96-8756-5ce365d2398d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.098099982Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/ff14cda9-79b4-4a96-8756-5ce365d2398d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.098137033Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ff14cda9-79b4-4a96-8756-5ce365d2398d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ff14cda9-79b4-4a96-8756-5ce365d2398d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.155891642Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"ff14cda9-79b4-4a96-8756-5ce365d2398d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ff14cda9-79b4-4a96-8756-5ce365d2398d\" \"--data\" \"/data/regions/ff14cda9-79b4-4a96-8756-5ce365d2398d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.157898472Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"ff14cda9-79b4-4a96-8756-5ce365d2398d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.231132753Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"region":"ff14cda9-79b4-4a96-8756-5ce365d2398d","component":"worker"}
{"msg":"region ff14cda9-79b4-4a96-8756-5ce365d2398d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.231175284Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.231303818Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ff14cda9-79b4-4a96-8756-5ce365d2398d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.238627177Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ff14cda9-79b4-4a96-8756-5ce365d2398d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.245433991Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.255140732Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ff14cda9-79b4-4a96-8756-5ce365d2398d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.258395599Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ff14cda9-79b4-4a96-8756-5ce365d2398d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.258701318Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.25876514Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.258817971Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.258867013Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.263347417Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.268546343Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:34.415867071Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions","method":"POST","req_id":"0faf88ec-a8c2-4b15-bba9-9d96c771d0dd","remote_addr":"[fd00:1122:3344:101::a]:47087","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":992,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.725129603Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14","method":"GET","req_id":"c0ecbc1a-dcc8-44ee-bdf2-da9c640326d5","remote_addr":"[fd00:1122:3344:101::a]:47087","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":50,"response_code":"200"}
{"msg":"path is \"/data/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.727934307Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"dataset is oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.736994078Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"snapshot is oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14@2bf29d11-a517-406d-9acd-40559ddbcdcb","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.745653707Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14\", \"2bf29d11-a517-406d-9acd-40559ddbcdcb\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.745686058Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.757495752Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14/snapshots/2bf29d11-a517-406d-9acd-40559ddbcdcb","method":"GET","req_id":"d2a3b8ce-6822-4c97-8c7d-0c4e86a531b7","remote_addr":"[fd00:1122:3344:101::a]:47087","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":29574,"response_code":"200"}
{"msg":"path is \"/data/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.760130211Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"dataset is oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.768011296Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"snapshot is oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14@2bf29d11-a517-406d-9acd-40559ddbcdcb","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.774855561Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14\", \"2bf29d11-a517-406d-9acd-40559ddbcdcb\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.774887502Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 02e8e590-f018-4ea9-900c-3f16aae4fb14-2bf29d11-a517-406d-9acd-40559ddbcdcb state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.783800139Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"applying SMF actions for region 02e8e590-f018-4ea9-900c-3f16aae4fb14 running snapshot 2bf29d11-a517-406d-9acd-40559ddbcdcb (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.783995655Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.784032736Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14/snapshots/2bf29d11-a517-406d-9acd-40559ddbcdcb/run","method":"POST","req_id":"288dea2f-2488-4602-8536-1302b6ed53d0","remote_addr":"[fd00:1122:3344:101::a]:47087","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":23925,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-02e8e590-f018-4ea9-900c-3f16aae4fb14-2bf29d11-a517-406d-9acd-40559ddbcdcb","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.786808659Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-02e8e590-f018-4ea9-900c-3f16aae4fb14-2bf29d11-a517-406d-9acd-40559ddbcdcb","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.789818099Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-02e8e590-f018-4ea9-900c-3f16aae4fb14-2bf29d11-a517-406d-9acd-40559ddbcdcb","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.800314803Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14/.zfs/snapshot/2bf29d11-a517-406d-9acd-40559ddbcdcb","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.800587861Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.800656253Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.800701775Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.800744106Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.800794917Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.80457207Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.809390055Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"worker"}
{"msg":"region 02e8e590-f018-4ea9-900c-3f16aae4fb14 running snapshot 2bf29d11-a517-406d-9acd-40559ddbcdcb state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:35.809404365Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"path is \"/data/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:36.084390664Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"dataset is oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:36.092456486Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"datafile"}
{"msg":"snapshot is oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14@2bf29d11-a517-406d-9acd-40559ddbcdcb","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:36.100487636Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_f7a83f52-88ee-4d82-bc3d-05b564dc479a/crucible/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14\", \"2bf29d11-a517-406d-9acd-40559ddbcdcb\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:36.100535488Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T14:43:36.108725813Z","hostname":"oxz_crucible_edab6306-aac4-43c4-9b7a-2cd88a117e9a","pid":22988,"uri":"/crucible/0/regions/02e8e590-f018-4ea9-900c-3f16aae4fb14/snapshots/2bf29d11-a517-406d-9acd-40559ddbcdcb/run","method":"POST","req_id":"480c4bfd-1e41-4e53-89e0-eae421cbaed7","remote_addr":"[fd00:1122:3344:101::a]:47087","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":25004,"response_code":"200"}