[ Nov 15 21:42:03 Enabled. ]
[ Nov 15 21:42:03 Rereading configuration. ]
[ Nov 15 21:42:05 Rereading configuration. ]
[ Nov 15 21:42:11 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_399573dc-9aa5-448e-8896-7bfbaad8acc9/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_399573dc-9aa5-448e-8896-7bfbaad8acc9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:11.115091911Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:11.118226162Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:11.118241622Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:11.134416867Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:11.394798637Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.823403581Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64388"}
{"msg":"region 4950f733-8ec2-49da-8af5-078305bc0548 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.849099588Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.853337945Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"uri":"/crucible/0/regions","method":"POST","req_id":"f0618028-5513-43f9-886e-ecb90958b940","remote_addr":"[fd00:1122:3344:101::b]:64388","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":14025,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.853785358Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_399573dc-9aa5-448e-8896-7bfbaad8acc9/crucible/regions/4950f733-8ec2-49da-8af5-078305bc0548","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.862178771Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_399573dc-9aa5-448e-8896-7bfbaad8acc9/crucible/regions/4950f733-8ec2-49da-8af5-078305bc0548","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.862209296Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"4950f733-8ec2-49da-8af5-078305bc0548\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/4950f733-8ec2-49da-8af5-078305bc0548\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.964878459Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"region":"4950f733-8ec2-49da-8af5-078305bc0548","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"4950f733-8ec2-49da-8af5-078305bc0548\" \"--data\" \"/data/regions/4950f733-8ec2-49da-8af5-078305bc0548\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.96500269Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"region":"4950f733-8ec2-49da-8af5-078305bc0548","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:29.987709827Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"uri":"/crucible/0/regions","method":"POST","req_id":"4a2a966c-a2d4-4d0f-8e7c-af4929cf4340","remote_addr":"[fd00:1122:3344:101::b]:64388","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":79,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.112243106Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"region":"4950f733-8ec2-49da-8af5-078305bc0548","component":"worker"}
{"msg":"region 4950f733-8ec2-49da-8af5-078305bc0548 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.112289586Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.112389817Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-4950f733-8ec2-49da-8af5-078305bc0548","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.112797329Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-4950f733-8ec2-49da-8af5-078305bc0548","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.124639116Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.141534127Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-4950f733-8ec2-49da-8af5-078305bc0548","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.145755043Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/4950f733-8ec2-49da-8af5-078305bc0548","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.146331627Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.14838378Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.14843532Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.148473141Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.158997949Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.163517238Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:30.613064546Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"uri":"/crucible/0/regions","method":"POST","req_id":"9830aea0-00c0-4541-8c0a-9e854ae9d2dd","remote_addr":"[fd00:1122:3344:101::b]:64388","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.191117878Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"uri":"/crucible/0/regions/4950f733-8ec2-49da-8af5-078305bc0548","method":"GET","req_id":"ac0539ed-bc42-404c-901d-60eb0c6a0eb3","remote_addr":"[fd00:1122:3344:101::b]:64388","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":91,"response_code":"200"}
{"msg":"path is \"/data/regions/4950f733-8ec2-49da-8af5-078305bc0548\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.194226168Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"datafile"}
{"msg":"dataset is oxp_399573dc-9aa5-448e-8896-7bfbaad8acc9/crucible/regions/4950f733-8ec2-49da-8af5-078305bc0548","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.206631548Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"datafile"}
{"msg":"region 4950f733-8ec2-49da-8af5-078305bc0548 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.218487295Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.218632256Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-4950f733-8ec2-49da-8af5-078305bc0548 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.219969425Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.220862481Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"uri":"/crucible/0/regions/4950f733-8ec2-49da-8af5-078305bc0548","method":"DELETE","req_id":"555292e7-858a-4e33-bdaf-725f0bade800","remote_addr":"[fd00:1122:3344:101::b]:64388","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26669,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.224679865Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"uri":"/crucible/0/regions/4950f733-8ec2-49da-8af5-078305bc0548","method":"GET","req_id":"f1c6cd34-4c22-47e3-af77-c3ab20fc0c90","remote_addr":"[fd00:1122:3344:101::b]:64388","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.226745449Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_399573dc-9aa5-448e-8896-7bfbaad8acc9/crucible/regions/4950f733-8ec2-49da-8af5-078305bc0548\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.239036009Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"region":"4950f733-8ec2-49da-8af5-078305bc0548","component":"worker"}
{"msg":"region 4950f733-8ec2-49da-8af5-078305bc0548 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.331968972Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-15T21:42:31.345652671Z","hostname":"oxz_crucible_68c7b7cc-2abf-457c-86ef-34c9e3f29f13","pid":23814,"uri":"/crucible/0/regions/4950f733-8ec2-49da-8af5-078305bc0548","method":"GET","req_id":"76ee2696-fe62-4954-bd45-4e150e044e7e","remote_addr":"[fd00:1122:3344:101::b]:64388","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":68,"response_code":"200"}