[ Jul 19 22:57:30 Enabled. ]
[ Jul 19 22:57:30 Rereading configuration. ]
[ Jul 19 22:57:33 Rereading configuration. ]
[ Jul 19 22:57:37 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_70ae9063-370d-4ed5-8466-7bd7367499ff/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_70ae9063-370d-4ed5-8466-7bd7367499ff/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:37.3397653Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:37.34523924Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:37.34525447Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:37.362976124Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:37.497563784Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.58430112Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:49246"}
{"msg":"region f2953735-060f-4e44-ade1-bc38a74014ef state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.587368286Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.589591376Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.589609237Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions","method":"POST","req_id":"c892aa97-ccf9-49d3-8402-87a0255ddb02","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":3094,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_70ae9063-370d-4ed5-8466-7bd7367499ff/crucible/regions/f2953735-060f-4e44-ade1-bc38a74014ef","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.599566328Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_70ae9063-370d-4ed5-8466-7bd7367499ff/crucible/regions/f2953735-060f-4e44-ade1-bc38a74014ef","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.599599809Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"f2953735-060f-4e44-ade1-bc38a74014ef\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f2953735-060f-4e44-ade1-bc38a74014ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.6499795Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"region":"f2953735-060f-4e44-ade1-bc38a74014ef","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f2953735-060f-4e44-ade1-bc38a74014ef\" \"--data\" \"/data/regions/f2953735-060f-4e44-ade1-bc38a74014ef\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.650026991Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"region":"f2953735-060f-4e44-ade1-bc38a74014ef","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.771325008Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"region":"f2953735-060f-4e44-ade1-bc38a74014ef","component":"worker"}
{"msg":"region f2953735-060f-4e44-ade1-bc38a74014ef state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.771375009Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.7714501Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-f2953735-060f-4e44-ade1-bc38a74014ef","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.771879158Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f2953735-060f-4e44-ade1-bc38a74014ef","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.775430143Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.78620299Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f2953735-060f-4e44-ade1-bc38a74014ef","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.791001278Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f2953735-060f-4e44-ade1-bc38a74014ef","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.791705711Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.791790652Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.791836073Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.791874443Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.796682542Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.801854466Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:54.897713377Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions","method":"POST","req_id":"5f17df49-9150-4330-b397-4ced6a8bb994","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":824,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.682785717Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions/f2953735-060f-4e44-ade1-bc38a74014ef","method":"GET","req_id":"8794a12c-c88a-4bb3-815f-08bb12197c18","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":95,"response_code":"200"}
{"msg":"path is \"/data/regions/f2953735-060f-4e44-ade1-bc38a74014ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.685099849Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"dataset is oxp_70ae9063-370d-4ed5-8466-7bd7367499ff/crucible/regions/f2953735-060f-4e44-ade1-bc38a74014ef","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.69441219Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"region f2953735-060f-4e44-ade1-bc38a74014ef state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.704467073Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.704604056Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.704674957Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions/f2953735-060f-4e44-ade1-bc38a74014ef","method":"DELETE","req_id":"65c02ade-cb58-41bf-b90b-980d3a065507","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":19624,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-f2953735-060f-4e44-ade1-bc38a74014ef (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.705665535Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.706813296Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions/f2953735-060f-4e44-ade1-bc38a74014ef","method":"GET","req_id":"b45116d9-f26e-4b97-80f3-f8fd92e5db7a","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.710141637Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_70ae9063-370d-4ed5-8466-7bd7367499ff/crucible/regions/f2953735-060f-4e44-ade1-bc38a74014ef\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.719373596Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"region":"f2953735-060f-4e44-ade1-bc38a74014ef","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.770212945Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions/f2953735-060f-4e44-ade1-bc38a74014ef","method":"GET","req_id":"108c0c58-9002-4ae0-8824-3901ec68edd2","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":607,"response_code":"200"}
{"msg":"region f2953735-060f-4e44-ade1-bc38a74014ef state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.770737705Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:55.875227114Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions/f2953735-060f-4e44-ade1-bc38a74014ef","method":"GET","req_id":"9d4c45b9-2466-4f99-bdd1-9480fc12b9e0","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"region ab7ff5e3-afac-42dc-83d7-b185aef2d3b1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.501026249Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.501213542Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.501280323Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions","method":"POST","req_id":"52eb3388-6e4f-403a-a2e4-37eaba47cc6d","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":258,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_70ae9063-370d-4ed5-8466-7bd7367499ff/crucible/regions/ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.512726753Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_70ae9063-370d-4ed5-8466-7bd7367499ff/crucible/regions/ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.512764024Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ab7ff5e3-afac-42dc-83d7-b185aef2d3b1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ab7ff5e3-afac-42dc-83d7-b185aef2d3b1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.595590017Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"region":"ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ab7ff5e3-afac-42dc-83d7-b185aef2d3b1\" \"--data\" \"/data/regions/ab7ff5e3-afac-42dc-83d7-b185aef2d3b1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.595675719Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"region":"ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.677975023Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"region":"ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","component":"worker"}
{"msg":"region ab7ff5e3-afac-42dc-83d7-b185aef2d3b1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.678227018Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.679096844Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.680490029Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.684980581Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.697264096Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.70133298Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ab7ff5e3-afac-42dc-83d7-b185aef2d3b1","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.701636666Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.701707867Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.701758178Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.701806569Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.707200237Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.713543693Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-07-19T22:57:57.817091306Z","hostname":"oxz_crucible_830480db-f849-4472-b905-ded0419ecd34","pid":24261,"uri":"/crucible/0/regions","method":"POST","req_id":"755e1b81-861e-4ceb-87d3-726711e7bc23","remote_addr":"[fd00:1122:3344:101::a]:49246","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":860,"response_code":"200"}