From 8e49cbc7151e6eeee8279c076021feb7c3f47b1d Mon Sep 17 00:00:00 2001 From: Philipp Zehnder Date: Wed, 26 Jun 2024 16:40:37 +0200 Subject: [PATCH] fix(#2954): data type change in adapter causes data storage failure in influxdb (#2958) * fix(#2954): Add test to reproduce problem * fix(#2954): Add warning message to data type change if adapter is edited --- .../support/utils/connect/ConnectBtns.ts | 12 +++++ .../{ => connect}/ConnectEventSchemaUtils.ts | 44 ++++++++++------- .../support/utils/connect/ConnectUtils.ts | 10 +++- .../support/utils/datalake/DataLakeBtns.ts | 23 +++++++++ .../support/utils/datalake/DataLakeUtils.ts | 17 +++++++ .../tests/adapter/editAdapter.smoke.spec.ts | 48 ++++++++++++++++++- .../editAdapterSettingsAndPipeline.spec.ts | 2 +- .../editAdapterValuesAndFields.spec.ts | 9 ++-- .../rules/deleteTransformationRule.spec.ts | 2 +- .../adapter/rules/schemaRules.smoke.spec.ts | 2 +- .../tests/adapter/rules/streamRules.spec.ts | 2 +- ui/cypress/tests/adapter/rules/valueRules.ts | 2 +- .../datalake-configuration.component.html | 4 +- .../edit-schema-transformation.component.html | 15 ++++++ .../edit-schema-transformation.component.scss | 2 + .../edit-schema-transformation.component.ts | 6 +++ .../edit-correction-value.component.scss | 8 +--- .../edit-event-property.component.scss | 8 ++++ 18 files changed, 179 insertions(+), 37 deletions(-) rename ui/cypress/support/utils/{ => connect}/ConnectEventSchemaUtils.ts (85%) create mode 100644 ui/cypress/support/utils/datalake/DataLakeBtns.ts diff --git a/ui/cypress/support/utils/connect/ConnectBtns.ts b/ui/cypress/support/utils/connect/ConnectBtns.ts index 0fab8973c0..f828bcf148 100644 --- a/ui/cypress/support/utils/connect/ConnectBtns.ts +++ b/ui/cypress/support/utils/connect/ConnectBtns.ts @@ -40,6 +40,18 @@ export class ConnectBtns { return cy.dataCy('store-edit-adapter'); } + public static changeRuntimeType() { + return cy.dataCy('connect-change-runtime-type', { timeout: 10000 }); + } + + public static updateAndMigratePipelines() { + return cy.dataCy('btn-update-adapter-migrate-pipelines'); + } + + public static nextBtn() { + return cy.get('button').contains('Next').parent(); + } + // ===================== Adapter settings btns ========================== public static adapterSettingsStartAdapter() { return cy.dataCy('adapter-settings-start-adapter-btn'); diff --git a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts b/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts similarity index 85% rename from ui/cypress/support/utils/ConnectEventSchemaUtils.ts rename to ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts index 663a42edbf..098ea68554 100644 --- a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts +++ b/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts @@ -1,21 +1,23 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ +import { ConnectBtns } from './ConnectBtns'; + export class ConnectEventSchemaUtils { public static markPropertyAsDimension(propertyName: string) { cy.dataCy('property-scope-' + propertyName, { timeout: 10000 }) @@ -196,9 +198,13 @@ export class ConnectEventSchemaUtils { public static changePropertyDataType( propertyName: string, dataType: string, + warningIsShown: boolean = false, ) { ConnectEventSchemaUtils.clickEditProperty(propertyName); - cy.dataCy('connect-change-runtime-type') + + this.checkIfWarningIsShown(warningIsShown); + + ConnectBtns.changeRuntimeType() .click() .get('mat-option') .contains(dataType) @@ -208,13 +214,19 @@ export class ConnectEventSchemaUtils { cy.dataCy('edit-' + propertyName, { timeout: 10000 }).click({ force: true, }); - cy.dataCy('connect-change-runtime-type', { timeout: 10000 }).contains( - dataType, - ); + ConnectBtns.changeRuntimeType().contains(dataType); cy.wait(1000); cy.dataCy('sp-save-edit-property').click(); } + private static checkIfWarningIsShown(warningIsShown: boolean) { + if (warningIsShown) { + cy.dataCy('warning-change-data-type').should('be.visible'); + } else { + cy.dataCy('warning-change-data-type').should('not.exist'); + } + } + public static eventSchemaNextBtnDisabled() { cy.get('#event-schema-next-button').should('be.disabled'); } diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts index 6fe6cfd708..fa30cd01e0 100644 --- a/ui/cypress/support/utils/connect/ConnectUtils.ts +++ b/ui/cypress/support/utils/connect/ConnectUtils.ts @@ -18,7 +18,7 @@ import { StaticPropertyUtils } from '../StaticPropertyUtils'; import { AdapterInput } from '../../model/AdapterInput'; -import { ConnectEventSchemaUtils } from '../ConnectEventSchemaUtils'; +import { ConnectEventSchemaUtils } from './ConnectEventSchemaUtils'; import { DataLakeUtils } from '../datalake/DataLakeUtils'; import { ConnectBtns } from './ConnectBtns'; import { AdapterBuilder } from '../../builder/AdapterBuilder'; @@ -208,6 +208,14 @@ export class ConnectUtils { ); } + public static storeAndStartEditedAdapter() { + ConnectUtils.finishEventSchemaConfiguration(); + ConnectBtns.storeEditAdapter().click(); + ConnectBtns.updateAndMigratePipelines().click(); + ConnectUtils.closeAdapterPreview(); + ConnectBtns.startAdapter().click(); + } + public static deleteAdapterAndAssociatedPipelines(switchUserCheck = false) { // Delete adapter and associated pipelines this.goToConnect(); diff --git a/ui/cypress/support/utils/datalake/DataLakeBtns.ts b/ui/cypress/support/utils/datalake/DataLakeBtns.ts new file mode 100644 index 0000000000..5bb7083108 --- /dev/null +++ b/ui/cypress/support/utils/datalake/DataLakeBtns.ts @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export class DataLakeBtns { + public static refreshDataLakeMeasures() { + return cy.dataCy('refresh-data-lake-measures'); + } +} diff --git a/ui/cypress/support/utils/datalake/DataLakeUtils.ts b/ui/cypress/support/utils/datalake/DataLakeUtils.ts index 1788212551..41fadcf9b0 100644 --- a/ui/cypress/support/utils/datalake/DataLakeUtils.ts +++ b/ui/cypress/support/utils/datalake/DataLakeUtils.ts @@ -332,4 +332,21 @@ export class DataLakeUtils { return currentDate; } + + public static waitForCountingResults() { + cy.dataCy('datalake-number-of-events-spinner', { + timeout: 10000, + }).should('exist'); + cy.dataCy('datalake-number-of-events-spinner', { + timeout: 10000, + }).should('not.exist'); + } + + public static getDatalakeNumberOfEvents(): Cypress.Chainable { + return cy + .dataCy('datalake-number-of-events', { timeout: 10000 }) + .should('be.visible') + .invoke('text') + .then(text => text.trim()); + } } diff --git a/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts b/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts index ba8a1f54e0..3becf859e8 100644 --- a/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts +++ b/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts @@ -19,15 +19,18 @@ import { ConnectUtils } from '../../support/utils/connect/ConnectUtils'; import { ConnectBtns } from '../../support/utils/connect/ConnectBtns'; import { AdapterBuilder } from '../../support/builder/AdapterBuilder'; +import { ConnectEventSchemaUtils } from '../../support/utils/connect/ConnectEventSchemaUtils'; +import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils'; +import { DataLakeBtns } from '../../support/utils/datalake/DataLakeBtns'; describe('Test Edit Adapter', () => { beforeEach('Setup Test', () => { // To set up test add a stream adapter that can be configured cy.initStreamPipesTest(); - ConnectUtils.addMachineDataSimulator('simulator'); }); it('Successfully edit adapter', () => { + ConnectUtils.addMachineDataSimulator('simulator'); const newAdapterName = 'Edited Adapter'; ConnectUtils.goToConnect(); @@ -68,4 +71,47 @@ describe('Test Edit Adapter', () => { cy.dataCy('adapter-name').contains(newAdapterName); cy.get('.sp-dialog-content').contains(newAdapterName); }); + + it('Successfully edit adapter with persistence pipeline', () => { + ConnectUtils.addMachineDataSimulator('simulator', true); + + ConnectUtils.goToConnect(); + + // stop adapter and edit adapter + ConnectBtns.stopAdapter().click(); + ConnectBtns.editAdapter().click(); + + // change data type of density to integer + ConnectBtns.nextBtn().click(); + ConnectEventSchemaUtils.changePropertyDataType( + 'density', + 'Integer', + true, + ); + + ConnectUtils.storeAndStartEditedAdapter(); + + // Validate that the data is further persisted in the database by checking if the amount of events in the data lake changes + DataLakeUtils.goToDatalakeConfiguration(); + + DataLakeUtils.waitForCountingResults(); + + let initialValue; + + DataLakeUtils.getDatalakeNumberOfEvents().then(value => { + initialValue = value; + }); + + DataLakeBtns.refreshDataLakeMeasures().click(); + + DataLakeUtils.waitForCountingResults(); + + DataLakeUtils.getDatalakeNumberOfEvents().then(newValue => { + // IMPORTANT: Currently we implemented a workaround by showing the user a warning message when the data type is changed. + // In the future, we need a migration mechanism to automatically change all the StreamPipes resources that are effected + // by the change. Once this is implemented the following line must be changed to .not.equal. + // The issue is tracked here: https://github.com/apache/streampipes/issues/2954 + expect(newValue).equal(initialValue); + }); + }); }); diff --git a/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts b/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts index 8b63cafac1..da5089a227 100644 --- a/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts +++ b/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts @@ -70,7 +70,7 @@ describe('Test Edit Adapter and Pipeline', () => { cy.dataCy('sp-connect-adapter-edit-warning', { timeout: 60000, }).should('be.visible'); - cy.dataCy('btn-update-adapter-migrate-pipelines').click(); + ConnectBtns.updateAndMigratePipelines().click(); ConnectUtils.closeAdapterPreview(); cy.wait(1000); diff --git a/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts b/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts index 60ce694b21..52269352a0 100644 --- a/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts +++ b/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts @@ -50,7 +50,7 @@ describe('Test Edit Adapter', () => { .clear() .type('http://schema.org/Numbers'); // Change field data type - cy.dataCy('connect-change-runtime-type') + ConnectBtns.changeRuntimeType() .click() .get('mat-option') .contains('Double') @@ -83,10 +83,7 @@ describe('Test Edit Adapter', () => { 'have.value', 'http://schema.org/Numbers', ); - cy.dataCy('connect-change-runtime-type').should( - 'include.text', - 'Double', - ); + ConnectBtns.changeRuntimeType().should('include.text', 'Double'); cy.dataCy('connect-schema-correction-value').should('have.value', '2'); cy.dataCy('connect-schema-correction-operator').should( 'include.text', @@ -96,7 +93,7 @@ describe('Test Edit Adapter', () => { // Delete inserted values in edit field cy.dataCy('connect-edit-field-runtime-name').clear(); cy.get('[id="domainproperty"]').clear(); - cy.dataCy('connect-change-runtime-type') + ConnectBtns.changeRuntimeType() .click() .get('mat-option') .contains('Float') diff --git a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts index 6ecb38744b..ad558ab8d2 100644 --- a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts +++ b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts @@ -17,7 +17,7 @@ */ import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils'; import { FileManagementUtils } from '../../../support/utils/FileManagementUtils'; -import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils'; +import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils'; import { AdapterBuilder } from '../../../support/builder/AdapterBuilder'; import { ConnectBtns } from '../../../support/utils/connect/ConnectBtns'; diff --git a/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts b/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts index 39462cfc06..1aa4d14ce6 100644 --- a/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts +++ b/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts @@ -18,7 +18,7 @@ import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils'; import { FileManagementUtils } from '../../../support/utils/FileManagementUtils'; -import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils'; +import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils'; describe('Connect schema rule transformations', () => { beforeEach('Setup Test', () => { diff --git a/ui/cypress/tests/adapter/rules/streamRules.spec.ts b/ui/cypress/tests/adapter/rules/streamRules.spec.ts index dbfab9dab7..bf9f946a59 100644 --- a/ui/cypress/tests/adapter/rules/streamRules.spec.ts +++ b/ui/cypress/tests/adapter/rules/streamRules.spec.ts @@ -18,7 +18,7 @@ import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils'; import { FileManagementUtils } from '../../../support/utils/FileManagementUtils'; -import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils'; +import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils'; describe('Connect aggregation rule transformations', () => { beforeEach('Setup Test', () => { diff --git a/ui/cypress/tests/adapter/rules/valueRules.ts b/ui/cypress/tests/adapter/rules/valueRules.ts index 60ffdbf21a..2163f454d2 100644 --- a/ui/cypress/tests/adapter/rules/valueRules.ts +++ b/ui/cypress/tests/adapter/rules/valueRules.ts @@ -18,7 +18,7 @@ import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils'; import { FileManagementUtils } from '../../../support/utils/FileManagementUtils'; -import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils'; +import { ConnectEventSchemaUtils } from '../../../support/utils/connect/ConnectEventSchemaUtils'; describe('Connect value rule transformations', () => { beforeEach('Setup Test', () => { diff --git a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html index b202b2955f..5044fd3091 100644 --- a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html +++ b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html @@ -36,7 +36,8 @@