Add a generic CSV importer

This commit is contained in:
brantje 2017-02-18 15:26:08 +01:00
parent 3dd9e84de7
commit 9e7f47a715
No known key found for this signature in database
GPG key ID: 5FF1D117F918687F
15 changed files with 516 additions and 110 deletions

View file

@ -74,7 +74,21 @@ class TranslationController extends ApiController {
'import.no.label' => $this->trans->t('Credential has no label, skipping'),
'import.adding' => $this->trans->t('Adding {{credential}}'),
'import.added' => $this->trans->t('Added {{credential}}'),
'import.skipping' => $this->trans->t('Skipping credential, missing label on line {{line}}'),
'import.loaded' => $this->trans->t('Parsed {{num}} credentials, starting to import'),
'import.importing' => $this->trans->t('Importing'),
'import.start' => $this->trans->t('Start import'),
'select.csv' => $this->trans->t('Select csv file'),
'parsed.csv.rows' => $this->trans->t('Parsed {{rows}} lines from csv file'),
'skip.first.row' => $this->trans->t('Skip first row'),
'import.csv.label.req' => $this->trans->t('You need to assign the label field before you can start the import.'),
'first.five.lines' => $this->trans->t('First 5 lines of the csv are shown.'),
'assign.column' => $this->trans->t('Assign the proper fields to each column.'),
'example.credential' => $this->trans->t('Example imported credential'),
'missing.importer' => $this->trans->t('Missing an importer? Try it with the generic csv importer.'),
'missing.importer.back' => $this->trans->t('Go back to importers.'),
// js/app/controllers/revision.js
'revision.deleted' => $this->trans->t('Revision deleted'),

View file

@ -892,6 +892,19 @@
.import-steps li {
list-style-type: disc; }
.import-table-outter {
overflow-x: scroll; }
.import-table {
padding-right: 15px; }
.import-table .inspect {
text-align: center;
width: 25px;
cursor: pointer; }
.import-table th, .import-table td {
text-align: left;
padding: 3px 5px; }
#app-settings-content:not(.ng-hide) {
height: 90px;
display: inherit !important;

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,209 @@
/**
* Nextcloud - passman
*
* @copyright Copyright (c) 2016, Sander Brand (brantje@gmail.com)
* @copyright Copyright (c) 2016, Marcos Zuriaga Miguel (wolfi@wolfi.es)
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
(function () {
'use strict';
/**
* @ngdoc function
* @name passmanApp.controller:MenuCtrl
* @description
* # MenuCtrl
* Controller of the passmanApp
*/
angular.module('passmanApp')
.controller('GenericCsvImportCtrl', ['$scope', 'CredentialService', '$translate',
function ($scope, CredentialService, $translate) {
$scope.hello = 'world';
$scope.credentialProperties = [
{
label: 'Label',
prop: 'label',
matching: ['label', 'title', 'name']
},
{
label: 'Username',
prop: 'username',
matching: ['username', 'user', 'login', 'login name']
},
{
label: 'Password',
prop: 'password',
matching: ['password', 'pass', 'pw']
},
{
label: 'TOTP Secret',
prop: 'otp',
matching: ['totp']
},
{
label: 'Custom field',
prop: 'custom_field'
},
{
label: 'Notes',
prop: 'description',
matching: ['notes', 'description', 'comments']
},
{
label: 'Email',
prop: 'email',
matching: ['email', 'mail']
},
{
label: 'URL',
prop: 'url',
matching: ['website', 'url', 'fulladdress', 'site', 'web site']
},
{
label: 'Tags',
prop: 'tags'
},
{
label: 'Ignored',
prop: null
}
];
var rowToCredential = function (row) {
var _credential = PassmanImporter.newCredential();
for(var k = 0; k < $scope.import_fields.length; k++){
var field = $scope.import_fields[k];
if(field){
if(field === 'otp'){
_credential.otp.secret = row[k]
} else if(field === 'custom_field'){
var key = ($scope.matched) ? $scope.parsed_csv[0][k] : 'Custom field '+ k;
_credential.custom_fields.push({
'label': key,
'value': row[k],
'secret': 0
})
} else if(field === 'tags'){
if( row[k]) {
console.log(row, k);
var tags = row[k].split(',');
console.log();
_credential.tags = tags.map(function (t) {
console.log(t);
return {text: t}
});
}
} else{
_credential[field] = row[k];
}
}
}
return _credential
};
$scope.inspectCredential = function (row) {
$scope.inspected_credential = rowToCredential(row);
};
$scope.csvLoaded = function (file) {
$scope.import_fields = [];
$scope.inspected_credential = false;
$scope.matched = false;
$scope.skipFirstRow = false;
var file_data = file.data.split(',');
file_data = decodeURIComponent(escape(window.atob(file_data[1])));
Papa.parse(file_data, {
complete: function(results) {
if(results.data) {
for(var i = 0; i < results.data[0].length; i++){
var propName = results.data[0][i];
$scope.import_fields[i] = null;
for(var p = 0; p < $scope.credentialProperties.length; p++){
var credentialProperty = $scope.credentialProperties[p];
if(credentialProperty.matching){
if(credentialProperty.matching.indexOf(propName.toLowerCase()) !== -1){
$scope.import_fields[i] = credentialProperty.prop;
$scope.matched = true;
}
}
}
}
if($scope.matched){
$scope.inspectCredential(results.data[1]);
}
$scope.parsed_csv = results.data;
$scope.$apply();
}
}
});
};
var addCredential = function (index) {
function handleState (index) {
if ($scope.parsed_csv[index + 1]) {
$scope.import_progress = {
progress: index / $scope.parsed_csv.length * 100,
loaded: index,
total: $scope.parsed_csv.length
};
addCredential(index + 1);
} else {
$scope.import_progress = {
progress: 100,
loaded: $scope.parsed_csv.length,
total: $scope.parsed_csv.length
};
$scope.log.push($translate.instant('done'));
$scope.importing = false;
}
}
var _credential = rowToCredential($scope.parsed_csv[index]);
_credential.vault_id = $scope.active_vault.vault_id;
if (!_credential.label) {
$scope.log.push($translate.instant('import.skipping', {line: index}));
handleState(index);
return;
}
$scope.log.push($translate.instant('import.adding', {credential: _credential.label}));
CredentialService.createCredential(_credential).then(function (result) {
if (result.credential_id) {
$scope.log.push($translate.instant('import.added', {credential: _credential.label}));
handleState(index);
}
});
};
$scope.importing = false;
$scope.startImport = function () {
$scope.importing = true;
$scope.log = [];
var start = ($scope.skipFirstRow) ? 1 : 0;
addCredential(start);
};
$scope.updateExample = function () {
var start = ($scope.skipFirstRow) ? 1 : 0;
$scope.inspectCredential($scope.parsed_csv[start]);
}
}]);
}());

View file

@ -36,6 +36,8 @@
function ($scope, $rootScope, SettingsService, VaultService, CredentialService, $location, $routeParams, $http, EncryptService, NotificationService, $sce, $translate) {
$scope.vault_settings = {};
$scope.new_vault_name = '';
$scope.showGenericImport = false;
$scope.active_vault = VaultService.getActiveVault();
if (!SettingsService.getSetting('defaultVault') || !SettingsService.getSetting('defaultVaultPass')) {
if (!$scope.active_vault) {

View file

@ -35,9 +35,9 @@
scope: {
autoScroll: '='
},
link: function postLink (scope) {
link: function postLink (scope, el) {
scope.$watch('autoScroll', function () {
$('#import_log').scrollTop($('#import_log')[0].scrollHeight);
$(el).scrollTop($(el)[0].scrollHeight);
}, true);
}
};

View file

@ -0,0 +1,47 @@
/**
* Nextcloud - passman
*
* @copyright Copyright (c) 2016, Sander Brand (brantje@gmail.com)
* @copyright Copyright (c) 2016, Marcos Zuriaga Miguel (wolfi@wolfi.es)
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
(function () {
'use strict';
/**
* @ngdoc directive
* @name passmanApp.directive:passwordGen
* @description
* # passwordGen
*/
angular.module('passmanApp')
.directive('credentialTemplate', [function () {
return {
templateUrl: 'views/partials/credential_template.html',
replace: true,
restrict: 'A',
scope: {
credential: '=credentialTemplate'
},
link: function (scope, element, attrs) {
console.log(attrs.showLabel)
scope.showLabel = (attrs.hasOwnProperty('showLabel'));
}
};
}]);
}());

View file

@ -53,11 +53,13 @@
fileReader.onprogress = function (event) {
var percent = (event.loaded / event.total * 100);
scope.$apply(scope.progress({
file_total: event.total,
file_loaded: event.loaded,
file_percent: percent
}));
if(scope.progress) {
scope.$apply(scope.progress({
file_total: event.total,
file_loaded: event.loaded,
file_percent: percent
}));
}
};
fileReader.onerror = function () {

File diff suppressed because one or more lines are too long

View file

@ -20,31 +20,50 @@
*
*/
.scan-result-table{
.scan-result-table {
margin-top: 10px;
.score{
.score {
padding-left: 0px;
padding-right: 15px;
}
}
.error{
.error {
color: #ce3702;
}
.import_log {
max-height: 600px;
overflow-y: auto;
textarea{
textarea {
width: 90%;
height: 200px;
}
}
.tab_container.settings{
.tab_container.settings {
margin-bottom: 50px;
}
.import-steps{
.import-steps {
padding-left: 16px;
li{
li {
list-style-type: disc;
}
margin-bottom: 10px;
}
.import-table-outter {
overflow-x: scroll;
}
.import-table {
padding-right: 15px;
.inspect{
text-align: center;
width: 25px;
cursor: pointer;
}
th, td {
text-align: left;
padding: 3px 5px;
}
}

View file

@ -45,6 +45,7 @@ script('passman', 'app/controllers/revision');
script('passman', 'app/controllers/settings');
script('passman', 'app/controllers/import');
script('passman', 'app/controllers/export');
script('passman', 'app/controllers/generic-csv-importer');
script('passman', 'app/filters/range');
script('passman', 'app/filters/propsfilter');
script('passman', 'app/filters/byte');
@ -76,6 +77,7 @@ script('passman', 'app/directives/clickselect');
script('passman', 'app/directives/colorfromstring');
script('passman', 'app/directives/credentialcounter');
script('passman', 'app/directives/clearbutton2');
script('passman', 'app/directives/credentialtemplate');
script('passman', 'importers/import-main');
script('passman', 'importers/importer-keepasscsv');
script('passman', 'importers/importer-lastpasscsv');

View file

@ -0,0 +1,96 @@
<div class="credential-data">
<div class="row" ng-show="credential.label && showLabel">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'label' | translate }}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field
value="credential.label"></span></div>
</div>
<div class="row" ng-show="credential.username">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'account' | translate }}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field
value="credential.username"></span></div>
</div>
<div class="row" ng-show="credential.password">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'password' | translate }}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
<span credential-field value="credential.password" secret="'true'">
</span>
</div>
</div>
<div class="row" ng-show="credential.otp.secret">
<div class="col-xs-4 col-md-3 col-lg-3">{{'otp' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span otp-generator
secret="credential.otp.secret"></span></div>
</div>
<div class="row" ng-show="credential.email">
<div class="col-xs-4 col-md-3 col-lg-3">{{'email' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field
value="credential.email"></span></div>
</div>
<div class="row" ng-show="credential.url">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'url' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field
value="credential.url"></span></div>
</div>
<div class="row" ng-show="credential.description">
<div class="col-xs-4 col-md-3 col-lg-3">{{'notes' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field value="credential.description_html"></span></div>
</div>
<div class="row" ng-show="credential.files.length > 0">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'files' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><div ng-repeat="file in credential.files"
class="link" ng-click="downloadFile(credential, file)">
{{file.filename}} ({{file.size | bytes}})
</div></div>
</div>
<div class="row" ng-repeat="field in credential.custom_fields">
<div class="col-xs-4 col-md-3 col-lg-3">{{field.label}}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
<span credential-field value="field.value" secret="field.secret" ng-if="field.field_type !== 'file' || !field.field_type"></span>
<span ng-if="field.field_type === 'file'" class="link" ng-click="downloadFile(credential, field.value)">{{field.value.filename}} ({{field.value.size | bytes}})</span>
</div>
</div>
<div class="row" ng-show="credential.expire_time > 0">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'expire.time' | translate }}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
{{credential.expire_time * 1000 | date:'dd-MM-yyyy @ HH:mm:ss'}}
</div>
</div>
<div class="row" ng-show="credential.changed">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'changed' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
{{credential.changed * 1000 | date:'dd-MM-yyyy @ HH:mm:ss'}}
</div>
</div>
<div class="row" ng-show="credential.created">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'created' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
{{credential.created * 1000 | date:'dd-MM-yyyy @ HH:mm:ss'}}
</div>
</div>
<div class="row">
<div class="col-xs-12">
<div class="tags">
<span class="tag" ng-repeat="tag in credential.tags track by $index">{{tag.text}}</span>
</div>
</div>
</div>
</div>

View file

@ -0,0 +1,68 @@
<div ng-controller="GenericCsvImportCtrl">
<div class="row">
<div class="col-xs-12 col-md-3">
<div>{{ 'select.csv' | translate}}
<input type="file" file-select accept=".csv"
success="csvLoaded">
</div>
<div ng-show="parsed_csv">
<span translate="parsed.csv.rows" translate-value-rows="{{ parsed_csv.length }}">
</span>
</div>
<div ng-show="parsed_csv">
<input type="checkbox" ng-model="skipFirstRow" ng-checked="matched"> {{ 'skip.first.row' | translate}}
</div>
<div ng-show="import_fields.indexOf('label') === -1 && parsed_csv">
<b>{{ 'import.csv.label.req' | translate}}</b>
</div>
<div ng-show="import_fields.indexOf('label') !== -1 && parsed_csv">
<button class="btn btn-success" ng-disabled="importing" ng-click="startImport()"><i class="fa fa-spinner fa-spin" ng-show="importing"></i> {{ (importing) ? ('import.importing' | translate) : ('import.start' | translate) }}</button>
</div>
<div>
<div ng-if="import_progress.progress > 0">
{{ 'upload.progress' | translate}}
<div progress-bar="import_progress.progress" index="import_progress.loaded" total="import_progress.total"></div>
</div>
</div>
<div>
<div ng-if="log" class="import_log">
<textarea id="import_log" auto-scroll="log">{{log.join('\n')}}</textarea>
</div>
</div>
</div>
<div class="col-xs-12 col-md-9" ng-show="parsed_csv">
<b>{{ 'first.five.lines' | translate }}</b><br />
{{ 'assign.column' | translate }}
<div class="import-table-outter">
<table class="import-table">
<tr ng-repeat="line in parsed_csv | limitTo:5">
<td class="inspect"><i class="fa fa-search"
ng-click="inspectCredential(line)"
ng-if="($index > 0 && matched && import_fields.length > 0) || ($index >= 0 && !matched && import_fields.length > 0)"></i>
</td>
<td ng-repeat="prop in line track by $index">
{{line[$index]}}
</td>
</tr>
<tr ng-repeat="line in parsed_csv | limitTo:1">
<td></td>
<td ng-repeat="prop in line track by $index">
<select ng-model="import_fields[$index]" ng-change="updateExample()"
ng-options="property.prop as property.label for property in credentialProperties">
</select>
</td>
</tr>
</table>
</div>
<div ng-show="inspected_credential && import_fields.length > 0">
<b>{{ 'example.credential' | translate}}</b>
<div credential-template="inspected_credential" show-label>
</div>
</div>
</div>
</div>
</div>

View file

@ -1,4 +1,11 @@
<div ng-controller="ImportCtrl">
<div>
<div ng-click="showGenericImport = !showGenericImport;" class="link">
<span ng-show="!showGenericImport">{{'missing.importer' | translate}}</span>
<span ng-show="showGenericImport">{{'missing.importer.back' | translate}}</span>
</div>
</div>
<div ng-controller="ImportCtrl" ng-show="!showGenericImport">
<div class="row">
<div class="col-xs-6">
<label>{{ 'import.type' | translate}}
@ -8,7 +15,9 @@
value="{{importer}}">
{{importer.name}}
</option>
</select></label>
</select>
</label>
<div ng-show="selectedImporter">
<b>{{ 'import.steps' | translate }}</b>
<ul class="import-steps">
@ -37,4 +46,5 @@
</div>
</div>
</div>
</div>
</div>
<div ng-include="'views/partials/forms/settings/generic_csv_import.html'" ng-show="showGenericImport"></div>

View file

@ -87,96 +87,8 @@
<span class="close icon-close" ng-click="closeSelected()"
alt="Close"></span>
<div class="credential-data">
<div class="row" ng-show="selectedCredential.username">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'account' | translate }}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field
value="selectedCredential.username"></span></div>
</div>
<div credential-template="selectedCredential">
<div class="row" ng-show="selectedCredential.password">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'password' | translate }}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
<span credential-field value="selectedCredential.password" secret="'true'">
</span>
</div>
</div>
<div class="row" ng-show="selectedCredential.otp.secret">
<div class="col-xs-4 col-md-3 col-lg-3">{{'otp' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span otp-generator
secret="selectedCredential.otp.secret"></span></div>
</div>
<div class="row" ng-show="selectedCredential.email">
<div class="col-xs-4 col-md-3 col-lg-3">{{'email' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field
value="selectedCredential.email"></span></div>
</div>
<div class="row" ng-show="selectedCredential.url">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'url' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field
value="selectedCredential.url"></span></div>
</div>
<div class="row" ng-show="selectedCredential.description">
<div class="col-xs-4 col-md-3 col-lg-3">{{'notes' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><span credential-field value="selectedCredential.description_html"></span></div>
</div>
<div class="row" ng-show="selectedCredential.files.length > 0">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'files' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9"><div ng-repeat="file in selectedCredential.files"
class="link" ng-click="downloadFile(selectedCredential, file)">
{{file.filename}} ({{file.size | bytes}})
</div></div>
</div>
<div class="row" ng-repeat="field in selectedCredential.custom_fields">
<div class="col-xs-4 col-md-3 col-lg-3">{{field.label}}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
<span credential-field value="field.value" secret="field.secret" ng-if="field.field_type !== 'file' || !field.field_type"></span>
<span ng-if="field.field_type === 'file'" class="link" ng-click="downloadFile(selectedCredential, field.value)">{{field.value.filename}} ({{field.value.size | bytes}})</span>
</div>
</div>
<div class="row" ng-show="selectedCredential.expire_time > 0">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'expire.time' | translate }}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
{{selectedCredential.expire_time * 1000 | date:'dd-MM-yyyy @ HH:mm:ss'}}
</div>
</div>
<div class="row" ng-show="selectedCredential.changed">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'changed' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
{{selectedCredential.changed * 1000 | date:'dd-MM-yyyy @ HH:mm:ss'}}
</div>
</div>
<div class="row" ng-show="selectedCredential.created">
<div class="col-xs-4 col-md-3 col-lg-3">{{ 'created' | translate}}</div>
<div class="col-xs-8 col-md-9 col-lg-9">
{{selectedCredential.created * 1000 | date:'dd-MM-yyyy @ HH:mm:ss'}}
</div>
</div>
<div class="row">
<div class="col-xs-12">
<div class="tags">
<span class="tag" ng-repeat="tag in selectedCredential.tags">{{tag.text}}</span>
</div>
</div>
</div>
</div>